nnetsauce

 1from .base.base import Base
 2from .base.baseRegressor import BaseRegressor
 3from .boosting.adaBoostClassifier import AdaBoostClassifier
 4from .custom.customClassifier import CustomClassifier
 5from .custom.customRegressor import CustomRegressor
 6from .deep.deepClassifier import DeepClassifier
 7from .deep.deepRegressor import DeepRegressor
 8from .deep.deepMTS import DeepMTS
 9from .glm.glmClassifier import GLMClassifier
10from .glm.glmRegressor import GLMRegressor
11from .lazypredict.lazyClassifier import LazyClassifier
12from .lazypredict.lazyRegressor import LazyRegressor
13from .lazypredict.lazyMTS import LazyMTS
14from .lazypredict.lazydeepClassifier import LazyDeepClassifier
15from .lazypredict.lazydeepRegressor import LazyDeepRegressor
16from .lazypredict.lazydeepMTS import LazyDeepMTS
17from .mts.mts import MTS
18from .multitask.multitaskClassifier import MultitaskClassifier
19from .multitask.simplemultitaskClassifier import SimpleMultitaskClassifier
20from .optimizers.optimizer import Optimizer
21from .randombag.randomBagClassifier import RandomBagClassifier
22from .randombag.randomBagRegressor import RandomBagRegressor
23from .ridge2.ridge2Classifier import Ridge2Classifier
24from .ridge2.ridge2Regressor import Ridge2Regressor
25from .ridge2.ridge2MultitaskClassifier import Ridge2MultitaskClassifier
26from .rvfl.bayesianrvflRegressor import BayesianRVFLRegressor
27from .rvfl.bayesianrvfl2Regressor import BayesianRVFL2Regressor
28from .sampling import SubSampler
29
30__all__ = [
31    "AdaBoostClassifier",
32    "Base",
33    "BaseRegressor",
34    "BayesianRVFLRegressor",
35    "BayesianRVFL2Regressor",
36    "CustomClassifier",
37    "CustomRegressor",
38    "DeepClassifier",
39    "DeepRegressor",
40    "DeepMTS",
41    "GLMClassifier",
42    "GLMRegressor",
43    "LazyClassifier",
44    "LazyRegressor",
45    "LazyMTS",
46    "LazyDeepClassifier",
47    "LazyDeepRegressor",
48    "LazyDeepMTS",
49    "MTS",
50    "MultitaskClassifier",
51    "SimpleMultitaskClassifier",
52    "Optimizer",
53    "RandomBagRegressor",
54    "RandomBagClassifier",
55    "Ridge2Regressor",
56    "Ridge2Classifier",
57    "Ridge2MultitaskClassifier",
58    "SubSampler",
59]
class AdaBoostClassifier(nnetsauce.boosting.bst.Boosting, sklearn.base.ClassifierMixin):
 19class AdaBoostClassifier(Boosting, ClassifierMixin):
 20    """AdaBoost Classification (SAMME) model class derived from class Boosting
 21
 22    Parameters:
 23
 24        obj: object
 25            any object containing a method fit (obj.fit()) and a method predict
 26            (obj.predict())
 27
 28        n_estimators: int
 29            number of boosting iterations
 30
 31        learning_rate: float
 32            learning rate of the boosting procedure
 33
 34        n_hidden_features: int
 35            number of nodes in the hidden layer
 36
 37        reg_lambda: float
 38            regularization parameter for weights
 39
 40        reg_alpha: float
 41            controls compromize between l1 and l2 norm of weights
 42
 43        activation_name: str
 44            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 45
 46        a: float
 47            hyperparameter for 'prelu' or 'elu' activation function
 48
 49        nodes_sim: str
 50            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 51            'uniform'
 52
 53        bias: boolean
 54            indicates if the hidden layer contains a bias term (True) or not
 55            (False)
 56
 57        dropout: float
 58            regularization parameter; (random) percentage of nodes dropped out
 59            of the training
 60
 61        direct_link: boolean
 62            indicates if the original predictors are included (True) in model's
 63            fitting or not (False)
 64
 65        n_clusters: int
 66            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 67                no clustering)
 68
 69        cluster_encode: bool
 70            defines how the variable containing clusters is treated (default is one-hot)
 71            if `False`, then labels are used, without one-hot encoding
 72
 73        type_clust: str
 74            type of clustering method: currently k-means ('kmeans') or Gaussian
 75            Mixture Model ('gmm')
 76
 77        type_scaling: a tuple of 3 strings
 78            scaling methods for inputs, hidden layer, and clustering respectively
 79            (and when relevant).
 80            Currently available: standardization ('std') or MinMax scaling ('minmax')
 81
 82        col_sample: float
 83            percentage of covariates randomly chosen for training
 84
 85        row_sample: float
 86            percentage of rows chosen for training, by stratified bootstrapping
 87
 88        seed: int
 89            reproducibility seed for nodes_sim=='uniform'
 90
 91        method: str
 92            type of Adaboost method, 'SAMME' (discrete) or 'SAMME.R' (real)
 93
 94        backend: str
 95            "cpu" or "gpu" or "tpu"
 96
 97    Attributes:
 98
 99        alpha_: list
100            AdaBoost coefficients alpha_m
101
102        base_learners_: dict
103            a dictionary containing the base learners
104
105    Examples:
106
107    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py)
108
109    ```python
110    import nnetsauce as ns
111    import numpy as np
112    from sklearn.datasets import load_breast_cancer
113    from sklearn.linear_model import LogisticRegression
114    from sklearn.model_selection import train_test_split
115    from sklearn import metrics
116    from time import time
117
118    breast_cancer = load_breast_cancer()
119    Z = breast_cancer.data
120    t = breast_cancer.target
121    np.random.seed(123)
122    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
123
124    # SAMME.R
125    clf = LogisticRegression(solver='liblinear', multi_class = 'ovr',
126                            random_state=123)
127    fit_obj = ns.AdaBoostClassifier(clf,
128                                    n_hidden_features=int(11.22338867),
129                                    direct_link=True,
130                                    n_estimators=250, learning_rate=0.01126343,
131                                    col_sample=0.72684326, row_sample=0.86429443,
132                                    dropout=0.63078613, n_clusters=2,
133                                    type_clust="gmm",
134                                    verbose=1, seed = 123,
135                                    method="SAMME.R")
136
137    start = time()
138    fit_obj.fit(X_train, y_train)
139    print(f"Elapsed {time() - start}")
140
141    start = time()
142    print(fit_obj.score(X_test, y_test))
143    print(f"Elapsed {time() - start}")
144
145    preds = fit_obj.predict(X_test)
146
147    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
148    print(metrics.classification_report(preds, y_test))
149
150    ```
151
152    """
153
154    # construct the object -----
155
156    def __init__(
157        self,
158        obj,
159        n_estimators=10,
160        learning_rate=0.1,
161        n_hidden_features=1,
162        reg_lambda=0,
163        reg_alpha=0.5,
164        activation_name="relu",
165        a=0.01,
166        nodes_sim="sobol",
167        bias=True,
168        dropout=0,
169        direct_link=False,
170        n_clusters=2,
171        cluster_encode=True,
172        type_clust="kmeans",
173        type_scaling=("std", "std", "std"),
174        col_sample=1,
175        row_sample=1,
176        seed=123,
177        verbose=1,
178        method="SAMME",
179        backend="cpu",
180    ):
181        super().__init__(
182            obj=obj,
183            n_estimators=n_estimators,
184            learning_rate=learning_rate,
185            n_hidden_features=n_hidden_features,
186            activation_name=activation_name,
187            a=a,
188            nodes_sim=nodes_sim,
189            bias=bias,
190            dropout=dropout,
191            direct_link=direct_link,
192            n_clusters=n_clusters,
193            cluster_encode=cluster_encode,
194            type_clust=type_clust,
195            type_scaling=type_scaling,
196            col_sample=col_sample,
197            row_sample=row_sample,
198            seed=seed,
199            backend=backend,
200        )
201
202        self.type_fit = "classification"
203        self.verbose = verbose
204        self.method = method
205        self.reg_lambda = reg_lambda
206        self.reg_alpha = reg_alpha
207
208        self.alpha_ = []
209        self.base_learners_ = dict.fromkeys(range(n_estimators))
210
211    def fit(self, X, y, sample_weight=None, **kwargs):
212        """Fit Boosting model to training data (X, y).
213
214        Parameters:
215
216            X: {array-like}, shape = [n_samples, n_features]
217                Training vectors, where n_samples is the number
218                of samples and n_features is the number of features.
219
220            y: array-like, shape = [n_samples]
221                Target values.
222
223            **kwargs: additional parameters to be passed to
224                    self.cook_training_set or self.obj.fit
225
226        Returns:
227
228             self: object
229        """
230
231        assert mx.is_factor(y), "y must contain only integers"
232
233        assert self.method in (
234            "SAMME",
235            "SAMME.R",
236        ), "`method` must be either 'SAMME' or 'SAMME.R'"
237
238        assert (self.reg_lambda <= 1) & (
239            self.reg_lambda >= 0
240        ), "must have self.reg_lambda <= 1 &  self.reg_lambda >= 0"
241
242        assert (self.reg_alpha <= 1) & (
243            self.reg_alpha >= 0
244        ), "must have self.reg_alpha <= 1 &  self.reg_alpha >= 0"
245
246        # training
247        n, p = X.shape
248        self.n_classes = len(np.unique(y))
249
250        if sample_weight is None:
251            w_m = np.repeat(1.0 / n, n)
252
253        else:
254            w_m = np.asarray(sample_weight)
255
256        base_learner = CustomClassifier(
257            self.obj,
258            n_hidden_features=self.n_hidden_features,
259            activation_name=self.activation_name,
260            a=self.a,
261            nodes_sim=self.nodes_sim,
262            bias=self.bias,
263            dropout=self.dropout,
264            direct_link=self.direct_link,
265            n_clusters=self.n_clusters,
266            type_clust=self.type_clust,
267            type_scaling=self.type_scaling,
268            col_sample=self.col_sample,
269            row_sample=self.row_sample,
270            seed=self.seed,
271        )
272
273        if self.verbose == 1:
274            pbar = Progbar(self.n_estimators)
275
276        if self.method == "SAMME":
277            err_m = 1e6
278            err_bound = 1 - 1 / self.n_classes
279            self.alpha_.append(1.0)
280            x_range_n = range(n)
281
282            for m in range(self.n_estimators):
283                preds = base_learner.fit(
284                    X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs
285                ).predict(X)
286
287                self.base_learners_.update(
288                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
289                )
290
291                cond = [y[i] != preds[i] for i in x_range_n]
292
293                err_m = max(
294                    sum([elt[0] * elt[1] for elt in zip(cond, w_m)]),
295                    2.220446049250313e-16,
296                )  # sum(w_m) == 1
297
298                if self.reg_lambda > 0:
299                    err_m += self.reg_lambda * (
300                        (1 - self.reg_alpha) * 0.5 * sum([x**2 for x in w_m])
301                        + self.reg_alpha * sum([abs(x) for x in w_m])
302                    )
303
304                err_m = min(err_m, err_bound)
305
306                alpha_m = self.learning_rate * log(
307                    (self.n_classes - 1) * (1 - err_m) / err_m
308                )
309
310                self.alpha_.append(alpha_m)
311
312                w_m_temp = [exp(alpha_m * cond[i]) for i in x_range_n]
313
314                sum_w_m = sum(w_m_temp)
315
316                w_m = np.asarray([w_m_temp[i] / sum_w_m for i in x_range_n])
317
318                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
319
320                if self.verbose == 1:
321                    pbar.update(m)
322
323            if self.verbose == 1:
324                pbar.update(self.n_estimators)
325
326            self.n_estimators = len(self.base_learners_)
327
328            return self
329
330        if self.method == "SAMME.R":
331            Y = mo.one_hot_encode2(y, self.n_classes)
332
333            if sample_weight is None:
334                w_m = np.repeat(1.0 / n, n)  # (N, 1)
335
336            else:
337                w_m = np.asarray(sample_weight)
338
339            for m in range(self.n_estimators):
340                probs = base_learner.fit(
341                    X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs
342                ).predict_proba(X)
343
344                np.clip(
345                    a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs
346                )
347
348                self.base_learners_.update(
349                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
350                )
351
352                w_m *= np.exp(
353                    -1.0
354                    * self.learning_rate
355                    * (1.0 - 1.0 / self.n_classes)
356                    * xlogy(Y, probs).sum(axis=1)
357                )
358
359                w_m /= np.sum(w_m)
360
361                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
362
363                if self.verbose == 1:
364                    pbar.update(m)
365
366            if self.verbose == 1:
367                pbar.update(self.n_estimators)
368
369            self.n_estimators = len(self.base_learners_)
370
371            return self
372
373    def predict(self, X, **kwargs):
374        """Predict test data X.
375
376        Parameters:
377
378            X: {array-like}, shape = [n_samples, n_features]
379                Training vectors, where n_samples is the number
380                of samples and n_features is the number of features.
381
382            **kwargs: additional parameters to be passed to
383                  self.cook_test_set
384
385        Returns:
386
387            model predictions: {array-like}
388        """
389
390        return self.predict_proba(X, **kwargs).argmax(axis=1)
391
392    def predict_proba(self, X, **kwargs):
393        """Predict probabilities for test data X.
394
395        Parameters:
396
397            X: {array-like}, shape = [n_samples, n_features]
398                Training vectors, where n_samples is the number
399                of samples and n_features is the number of features.
400
401            **kwargs: additional parameters to be passed to
402                  self.cook_test_set
403
404        Returns:
405
406            probability estimates for test data: {array-like}
407
408        """
409
410        n_iter = len(self.base_learners_)
411
412        if self.method == "SAMME":
413            ensemble_learner = np.zeros((X.shape[0], self.n_classes))
414
415            # if self.verbose == 1:
416            #    pbar = Progbar(n_iter)
417
418            for idx, base_learner in self.base_learners_.items():
419                preds = base_learner.predict(X, **kwargs)
420
421                ensemble_learner += self.alpha_[idx] * mo.one_hot_encode2(
422                    preds, self.n_classes
423                )
424
425                # if self.verbose == 1:
426                #    pbar.update(idx)
427
428            # if self.verbose == 1:
429            #    pbar.update(n_iter)
430
431            expit_ensemble_learner = expit(ensemble_learner)
432
433            sum_ensemble = expit_ensemble_learner.sum(axis=1)
434
435            return expit_ensemble_learner / sum_ensemble[:, None]
436
437        # if self.method == "SAMME.R":
438        ensemble_learner = 0
439
440        # if self.verbose == 1:
441        #    pbar = Progbar(n_iter)
442
443        for idx, base_learner in self.base_learners_.items():
444            probs = base_learner.predict_proba(X, **kwargs)
445
446            np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs)
447
448            log_preds_proba = np.log(probs)
449
450            ensemble_learner += (
451                log_preds_proba - log_preds_proba.mean(axis=1)[:, None]
452            )
453
454            # if self.verbose == 1:
455            #    pbar.update(idx)
456
457        ensemble_learner *= self.n_classes - 1
458
459        # if self.verbose == 1:
460        #    pbar.update(n_iter)
461
462        expit_ensemble_learner = expit(ensemble_learner)
463
464        sum_ensemble = expit_ensemble_learner.sum(axis=1)
465
466        return expit_ensemble_learner / sum_ensemble[:, None]
467
468    def score(self, X, y, scoring=None, **kwargs):
469        """Score the model on test set features X and response y.
470
471        Parameters:
472
473            X: {array-like}, shape = [n_samples, n_features]
474                Training vectors, where n_samples is the number
475                of samples and n_features is the number of features
476
477            y: array-like, shape = [n_samples]
478                Target values
479
480            scoring: str
481                must be in ('accuracy', 'average_precision',
482                           'brier_score_loss', 'f1', 'f1_micro',
483                           'f1_macro', 'f1_weighted',  'f1_samples',
484                           'neg_log_loss', 'precision', 'recall',
485                           'roc_auc')
486
487            **kwargs: additional parameters to be passed to scoring functions
488
489        Returns:
490
491            model scores: {array-like}
492
493        """
494
495        preds = self.predict(X)
496
497        if scoring is None:
498            scoring = "accuracy"
499
500        # check inputs
501        assert scoring in (
502            "accuracy",
503            "average_precision",
504            "brier_score_loss",
505            "f1",
506            "f1_micro",
507            "f1_macro",
508            "f1_weighted",
509            "f1_samples",
510            "neg_log_loss",
511            "precision",
512            "recall",
513            "roc_auc",
514        ), "'scoring' should be in ('accuracy', 'average_precision', \
515                           'brier_score_loss', 'f1', 'f1_micro', \
516                           'f1_macro', 'f1_weighted',  'f1_samples', \
517                           'neg_log_loss', 'precision', 'recall', \
518                           'roc_auc')"
519
520        scoring_options = {
521            "accuracy": skm2.accuracy_score,
522            "average_precision": skm2.average_precision_score,
523            "brier_score_loss": skm2.brier_score_loss,
524            "f1": skm2.f1_score,
525            "f1_micro": skm2.f1_score,
526            "f1_macro": skm2.f1_score,
527            "f1_weighted": skm2.f1_score,
528            "f1_samples": skm2.f1_score,
529            "neg_log_loss": skm2.log_loss,
530            "precision": skm2.precision_score,
531            "recall": skm2.recall_score,
532            "roc_auc": skm2.roc_auc_score,
533        }
534
535        return scoring_options[scoring](y, preds, **kwargs)

AdaBoost Classification (SAMME) model class derived from class Boosting

Parameters:

obj: object
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict())

n_estimators: int
    number of boosting iterations

learning_rate: float
    learning rate of the boosting procedure

n_hidden_features: int
    number of nodes in the hidden layer

reg_lambda: float
    regularization parameter for weights

reg_alpha: float
    controls compromize between l1 and l2 norm of weights

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform'

method: str
    type of Adaboost method, 'SAMME' (discrete) or 'SAMME.R' (real)

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

alpha_: list
    AdaBoost coefficients alpha_m

base_learners_: dict
    a dictionary containing the base learners

Examples:

See also https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py

import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time

breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)

# SAMME.R
clf = LogisticRegression(solver='liblinear', multi_class = 'ovr',
                        random_state=123)
fit_obj = ns.AdaBoostClassifier(clf,
                                n_hidden_features=int(11.22338867),
                                direct_link=True,
                                n_estimators=250, learning_rate=0.01126343,
                                col_sample=0.72684326, row_sample=0.86429443,
                                dropout=0.63078613, n_clusters=2,
                                type_clust="gmm",
                                verbose=1, seed = 123,
                                method="SAMME.R")

start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

start = time()
print(fit_obj.score(X_test, y_test))
print(f"Elapsed {time() - start}")

preds = fit_obj.predict(X_test)

print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
print(metrics.classification_report(preds, y_test))
AdaBoostClassifier( obj, n_estimators=10, learning_rate=0.1, n_hidden_features=1, reg_lambda=0, reg_alpha=0.5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=False, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, verbose=1, method='SAMME', backend='cpu')
156    def __init__(
157        self,
158        obj,
159        n_estimators=10,
160        learning_rate=0.1,
161        n_hidden_features=1,
162        reg_lambda=0,
163        reg_alpha=0.5,
164        activation_name="relu",
165        a=0.01,
166        nodes_sim="sobol",
167        bias=True,
168        dropout=0,
169        direct_link=False,
170        n_clusters=2,
171        cluster_encode=True,
172        type_clust="kmeans",
173        type_scaling=("std", "std", "std"),
174        col_sample=1,
175        row_sample=1,
176        seed=123,
177        verbose=1,
178        method="SAMME",
179        backend="cpu",
180    ):
181        super().__init__(
182            obj=obj,
183            n_estimators=n_estimators,
184            learning_rate=learning_rate,
185            n_hidden_features=n_hidden_features,
186            activation_name=activation_name,
187            a=a,
188            nodes_sim=nodes_sim,
189            bias=bias,
190            dropout=dropout,
191            direct_link=direct_link,
192            n_clusters=n_clusters,
193            cluster_encode=cluster_encode,
194            type_clust=type_clust,
195            type_scaling=type_scaling,
196            col_sample=col_sample,
197            row_sample=row_sample,
198            seed=seed,
199            backend=backend,
200        )
201
202        self.type_fit = "classification"
203        self.verbose = verbose
204        self.method = method
205        self.reg_lambda = reg_lambda
206        self.reg_alpha = reg_alpha
207
208        self.alpha_ = []
209        self.base_learners_ = dict.fromkeys(range(n_estimators))
type_fit
verbose
method
reg_lambda
reg_alpha
alpha_
base_learners_
def fit(self, X, y, sample_weight=None, **kwargs):
211    def fit(self, X, y, sample_weight=None, **kwargs):
212        """Fit Boosting model to training data (X, y).
213
214        Parameters:
215
216            X: {array-like}, shape = [n_samples, n_features]
217                Training vectors, where n_samples is the number
218                of samples and n_features is the number of features.
219
220            y: array-like, shape = [n_samples]
221                Target values.
222
223            **kwargs: additional parameters to be passed to
224                    self.cook_training_set or self.obj.fit
225
226        Returns:
227
228             self: object
229        """
230
231        assert mx.is_factor(y), "y must contain only integers"
232
233        assert self.method in (
234            "SAMME",
235            "SAMME.R",
236        ), "`method` must be either 'SAMME' or 'SAMME.R'"
237
238        assert (self.reg_lambda <= 1) & (
239            self.reg_lambda >= 0
240        ), "must have self.reg_lambda <= 1 &  self.reg_lambda >= 0"
241
242        assert (self.reg_alpha <= 1) & (
243            self.reg_alpha >= 0
244        ), "must have self.reg_alpha <= 1 &  self.reg_alpha >= 0"
245
246        # training
247        n, p = X.shape
248        self.n_classes = len(np.unique(y))
249
250        if sample_weight is None:
251            w_m = np.repeat(1.0 / n, n)
252
253        else:
254            w_m = np.asarray(sample_weight)
255
256        base_learner = CustomClassifier(
257            self.obj,
258            n_hidden_features=self.n_hidden_features,
259            activation_name=self.activation_name,
260            a=self.a,
261            nodes_sim=self.nodes_sim,
262            bias=self.bias,
263            dropout=self.dropout,
264            direct_link=self.direct_link,
265            n_clusters=self.n_clusters,
266            type_clust=self.type_clust,
267            type_scaling=self.type_scaling,
268            col_sample=self.col_sample,
269            row_sample=self.row_sample,
270            seed=self.seed,
271        )
272
273        if self.verbose == 1:
274            pbar = Progbar(self.n_estimators)
275
276        if self.method == "SAMME":
277            err_m = 1e6
278            err_bound = 1 - 1 / self.n_classes
279            self.alpha_.append(1.0)
280            x_range_n = range(n)
281
282            for m in range(self.n_estimators):
283                preds = base_learner.fit(
284                    X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs
285                ).predict(X)
286
287                self.base_learners_.update(
288                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
289                )
290
291                cond = [y[i] != preds[i] for i in x_range_n]
292
293                err_m = max(
294                    sum([elt[0] * elt[1] for elt in zip(cond, w_m)]),
295                    2.220446049250313e-16,
296                )  # sum(w_m) == 1
297
298                if self.reg_lambda > 0:
299                    err_m += self.reg_lambda * (
300                        (1 - self.reg_alpha) * 0.5 * sum([x**2 for x in w_m])
301                        + self.reg_alpha * sum([abs(x) for x in w_m])
302                    )
303
304                err_m = min(err_m, err_bound)
305
306                alpha_m = self.learning_rate * log(
307                    (self.n_classes - 1) * (1 - err_m) / err_m
308                )
309
310                self.alpha_.append(alpha_m)
311
312                w_m_temp = [exp(alpha_m * cond[i]) for i in x_range_n]
313
314                sum_w_m = sum(w_m_temp)
315
316                w_m = np.asarray([w_m_temp[i] / sum_w_m for i in x_range_n])
317
318                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
319
320                if self.verbose == 1:
321                    pbar.update(m)
322
323            if self.verbose == 1:
324                pbar.update(self.n_estimators)
325
326            self.n_estimators = len(self.base_learners_)
327
328            return self
329
330        if self.method == "SAMME.R":
331            Y = mo.one_hot_encode2(y, self.n_classes)
332
333            if sample_weight is None:
334                w_m = np.repeat(1.0 / n, n)  # (N, 1)
335
336            else:
337                w_m = np.asarray(sample_weight)
338
339            for m in range(self.n_estimators):
340                probs = base_learner.fit(
341                    X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs
342                ).predict_proba(X)
343
344                np.clip(
345                    a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs
346                )
347
348                self.base_learners_.update(
349                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
350                )
351
352                w_m *= np.exp(
353                    -1.0
354                    * self.learning_rate
355                    * (1.0 - 1.0 / self.n_classes)
356                    * xlogy(Y, probs).sum(axis=1)
357                )
358
359                w_m /= np.sum(w_m)
360
361                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
362
363                if self.verbose == 1:
364                    pbar.update(m)
365
366            if self.verbose == 1:
367                pbar.update(self.n_estimators)
368
369            self.n_estimators = len(self.base_learners_)
370
371            return self

Fit Boosting model to training data (X, y).

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

 self: object
def predict(self, X, **kwargs):
373    def predict(self, X, **kwargs):
374        """Predict test data X.
375
376        Parameters:
377
378            X: {array-like}, shape = [n_samples, n_features]
379                Training vectors, where n_samples is the number
380                of samples and n_features is the number of features.
381
382            **kwargs: additional parameters to be passed to
383                  self.cook_test_set
384
385        Returns:
386
387            model predictions: {array-like}
388        """
389
390        return self.predict_proba(X, **kwargs).argmax(axis=1)

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
      self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
392    def predict_proba(self, X, **kwargs):
393        """Predict probabilities for test data X.
394
395        Parameters:
396
397            X: {array-like}, shape = [n_samples, n_features]
398                Training vectors, where n_samples is the number
399                of samples and n_features is the number of features.
400
401            **kwargs: additional parameters to be passed to
402                  self.cook_test_set
403
404        Returns:
405
406            probability estimates for test data: {array-like}
407
408        """
409
410        n_iter = len(self.base_learners_)
411
412        if self.method == "SAMME":
413            ensemble_learner = np.zeros((X.shape[0], self.n_classes))
414
415            # if self.verbose == 1:
416            #    pbar = Progbar(n_iter)
417
418            for idx, base_learner in self.base_learners_.items():
419                preds = base_learner.predict(X, **kwargs)
420
421                ensemble_learner += self.alpha_[idx] * mo.one_hot_encode2(
422                    preds, self.n_classes
423                )
424
425                # if self.verbose == 1:
426                #    pbar.update(idx)
427
428            # if self.verbose == 1:
429            #    pbar.update(n_iter)
430
431            expit_ensemble_learner = expit(ensemble_learner)
432
433            sum_ensemble = expit_ensemble_learner.sum(axis=1)
434
435            return expit_ensemble_learner / sum_ensemble[:, None]
436
437        # if self.method == "SAMME.R":
438        ensemble_learner = 0
439
440        # if self.verbose == 1:
441        #    pbar = Progbar(n_iter)
442
443        for idx, base_learner in self.base_learners_.items():
444            probs = base_learner.predict_proba(X, **kwargs)
445
446            np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs)
447
448            log_preds_proba = np.log(probs)
449
450            ensemble_learner += (
451                log_preds_proba - log_preds_proba.mean(axis=1)[:, None]
452            )
453
454            # if self.verbose == 1:
455            #    pbar.update(idx)
456
457        ensemble_learner *= self.n_classes - 1
458
459        # if self.verbose == 1:
460        #    pbar.update(n_iter)
461
462        expit_ensemble_learner = expit(ensemble_learner)
463
464        sum_ensemble = expit_ensemble_learner.sum(axis=1)
465
466        return expit_ensemble_learner / sum_ensemble[:, None]

Predict probabilities for test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
      self.cook_test_set

Returns:

probability estimates for test data: {array-like}
def score(self, X, y, scoring=None, **kwargs):
468    def score(self, X, y, scoring=None, **kwargs):
469        """Score the model on test set features X and response y.
470
471        Parameters:
472
473            X: {array-like}, shape = [n_samples, n_features]
474                Training vectors, where n_samples is the number
475                of samples and n_features is the number of features
476
477            y: array-like, shape = [n_samples]
478                Target values
479
480            scoring: str
481                must be in ('accuracy', 'average_precision',
482                           'brier_score_loss', 'f1', 'f1_micro',
483                           'f1_macro', 'f1_weighted',  'f1_samples',
484                           'neg_log_loss', 'precision', 'recall',
485                           'roc_auc')
486
487            **kwargs: additional parameters to be passed to scoring functions
488
489        Returns:
490
491            model scores: {array-like}
492
493        """
494
495        preds = self.predict(X)
496
497        if scoring is None:
498            scoring = "accuracy"
499
500        # check inputs
501        assert scoring in (
502            "accuracy",
503            "average_precision",
504            "brier_score_loss",
505            "f1",
506            "f1_micro",
507            "f1_macro",
508            "f1_weighted",
509            "f1_samples",
510            "neg_log_loss",
511            "precision",
512            "recall",
513            "roc_auc",
514        ), "'scoring' should be in ('accuracy', 'average_precision', \
515                           'brier_score_loss', 'f1', 'f1_micro', \
516                           'f1_macro', 'f1_weighted',  'f1_samples', \
517                           'neg_log_loss', 'precision', 'recall', \
518                           'roc_auc')"
519
520        scoring_options = {
521            "accuracy": skm2.accuracy_score,
522            "average_precision": skm2.average_precision_score,
523            "brier_score_loss": skm2.brier_score_loss,
524            "f1": skm2.f1_score,
525            "f1_micro": skm2.f1_score,
526            "f1_macro": skm2.f1_score,
527            "f1_weighted": skm2.f1_score,
528            "f1_samples": skm2.f1_score,
529            "neg_log_loss": skm2.log_loss,
530            "precision": skm2.precision_score,
531            "recall": skm2.recall_score,
532            "roc_auc": skm2.roc_auc_score,
533        }
534
535        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('accuracy', 'average_precision',
               'brier_score_loss', 'f1', 'f1_micro',
               'f1_macro', 'f1_weighted',  'f1_samples',
               'neg_log_loss', 'precision', 'recall',
               'roc_auc')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.boosting.bst.Boosting
obj
n_estimators
learning_rate
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
class Base(sklearn.base.BaseEstimator):
 31class Base(BaseEstimator):
 32    """Base model from which all the other classes inherit.
 33
 34    This class contains the most important data preprocessing/feature engineering methods.
 35
 36    Parameters:
 37
 38        n_hidden_features: int
 39            number of nodes in the hidden layer
 40
 41        activation_name: str
 42            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 43
 44        a: float
 45            hyperparameter for 'prelu' or 'elu' activation function
 46
 47        nodes_sim: str
 48            type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
 49            'uniform'
 50
 51        bias: boolean
 52            indicates if the hidden layer contains a bias term (True) or
 53            not (False)
 54
 55        dropout: float
 56            regularization parameter; (random) percentage of nodes dropped out
 57            of the training
 58
 59        direct_link: boolean
 60            indicates if the original features are included (True) in model's
 61            fitting or not (False)
 62
 63        n_clusters: int
 64            number of clusters for type_clust='kmeans' or type_clust='gmm'
 65            clustering (could be 0: no clustering)
 66
 67        cluster_encode: bool
 68            defines how the variable containing clusters is treated (default is one-hot);
 69            if `False`, then labels are used, without one-hot encoding
 70
 71        type_clust: str
 72            type of clustering method: currently k-means ('kmeans') or Gaussian
 73            Mixture Model ('gmm')
 74
 75        type_scaling: a tuple of 3 strings
 76            scaling methods for inputs, hidden layer, and clustering respectively
 77            (and when relevant).
 78            Currently available: standardization ('std') or MinMax scaling ('minmax') or robust scaling ('robust')
 79
 80        col_sample: float
 81            percentage of features randomly chosen for training
 82
 83        row_sample: float
 84            percentage of rows chosen for training, by stratified bootstrapping
 85
 86        seed: int
 87            reproducibility seed for nodes_sim=='uniform', clustering and dropout
 88
 89        backend: str
 90            "cpu" or "gpu" or "tpu"
 91
 92    """
 93
 94    # construct the object -----
 95
 96    def __init__(
 97        self,
 98        n_hidden_features=5,
 99        activation_name="relu",
100        a=0.01,
101        nodes_sim="sobol",
102        bias=True,
103        dropout=0,
104        direct_link=True,
105        n_clusters=2,
106        cluster_encode=True,
107        type_clust="kmeans",
108        type_scaling=("std", "std", "std"),
109        col_sample=1,
110        row_sample=1,
111        seed=123,
112        backend="cpu",
113    ):
114        # input checks -----
115
116        sys_platform = platform.system()
117
118        if (sys_platform == "Windows") and (backend in ("gpu", "tpu")):
119            warnings.warn(
120                "No GPU/TPU computing on Windows yet, backend set to 'cpu'"
121            )
122            backend = "cpu"
123
124        assert activation_name in (
125            "relu",
126            "tanh",
127            "sigmoid",
128            "prelu",
129            "elu",
130        ), "'activation_name' must be in ('relu', 'tanh', 'sigmoid','prelu', 'elu')"
131
132        assert nodes_sim in (
133            "sobol",
134            "hammersley",
135            "uniform",
136            "halton",
137        ), "'nodes_sim' must be in ('sobol', 'hammersley', 'uniform', 'halton')"
138
139        assert type_clust in (
140            "kmeans",
141            "gmm",
142        ), "'type_clust' must be in ('kmeans', 'gmm')"
143
144        assert (len(type_scaling) == 3) & all(
145            type_scaling[i] in ("minmax", "std", "robust")
146            for i in range(len(type_scaling))
147        ), "'type_scaling' must have length 3, and available scaling methods are 'minmax' scaling, standardization ('std') and robust scaling ('robust')"
148
149        assert (col_sample >= 0) & (
150            col_sample <= 1
151        ), "'col_sample' must be comprised between 0 and 1 (both included)"
152
153        assert backend in (
154            "cpu",
155            "gpu",
156            "tpu",
157        ), "must have 'backend' in ('cpu', 'gpu', 'tpu')"
158
159        self.n_hidden_features = n_hidden_features
160        self.activation_name = activation_name
161        self.a = a
162        self.nodes_sim = nodes_sim
163        self.bias = bias
164        self.seed = seed
165        self.backend = backend
166        self.dropout = dropout
167        self.direct_link = direct_link
168        self.cluster_encode = cluster_encode
169        self.type_clust = type_clust
170        self.type_scaling = type_scaling
171        self.col_sample = col_sample
172        self.row_sample = row_sample
173        self.n_clusters = n_clusters
174        self.subsampler_ = None
175        self.index_col_ = None
176        self.index_row_ = True
177        self.clustering_obj_ = None
178        self.clustering_scaler_ = None
179        self.nn_scaler_ = None
180        self.scaler_ = None
181        self.encoder_ = None
182        self.W_ = None
183        self.X_ = None
184        self.y_ = None
185        self.y_mean_ = None
186        self.beta_ = None
187
188        # activation function -----
189        if sys_platform in ("Linux", "Darwin"):
190            activation_options = {
191                "relu": ac.relu if (self.backend == "cpu") else jnn.relu,
192                "tanh": np.tanh if (self.backend == "cpu") else jnp.tanh,
193                "sigmoid": (
194                    ac.sigmoid if (self.backend == "cpu") else jnn.sigmoid
195                ),
196                "prelu": partial(ac.prelu, a=a),
197                "elu": (
198                    partial(ac.elu, a=a)
199                    if (self.backend == "cpu")
200                    else partial(jnn.elu, a=a)
201                ),
202            }
203        else:  # on Windows currently, no JAX
204            activation_options = {
205                "relu": (
206                    ac.relu if (self.backend == "cpu") else NotImplementedError
207                ),
208                "tanh": (
209                    np.tanh if (self.backend == "cpu") else NotImplementedError
210                ),
211                "sigmoid": (
212                    ac.sigmoid
213                    if (self.backend == "cpu")
214                    else NotImplementedError
215                ),
216                "prelu": partial(ac.prelu, a=a),
217                "elu": (
218                    partial(ac.elu, a=a)
219                    if (self.backend == "cpu")
220                    else NotImplementedError
221                ),
222            }
223        self.activation_func = activation_options[activation_name]
224
225    # "preprocessing" methods to be inherited -----
226
227    def encode_clusters(self, X=None, predict=False, **kwargs):  #
228        """Create new covariates with kmeans or GMM clustering
229
230        Parameters:
231
232            X: {array-like}, shape = [n_samples, n_features]
233                Training vectors, where n_samples is the number
234                of samples and n_features is the number of features.
235
236            predict: boolean
237                is False on training set and True on test set
238
239            **kwargs:
240                additional parameters to be passed to the
241                clustering method
242
243        Returns:
244
245            Clusters' matrix, one-hot encoded: {array-like}
246
247        """
248
249        np.random.seed(self.seed)
250
251        if X is None:
252            X = self.X_
253
254        if isinstance(X, pd.DataFrame):
255            X = copy.deepcopy(X.values.astype(float))
256
257        if predict is False:  # encode training set
258            # scale input data before clustering
259            self.clustering_scaler_, scaled_X = mo.scale_covariates(
260                X, choice=self.type_scaling[2]
261            )
262
263            self.clustering_obj_, X_clustered = mo.cluster_covariates(
264                scaled_X,
265                self.n_clusters,
266                self.seed,
267                type_clust=self.type_clust,
268                **kwargs
269            )
270
271            if self.cluster_encode == True:
272                return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
273                    np.float16
274                )
275
276            return X_clustered.astype(np.float16)
277
278        # if predict == True, encode test set
279        X_clustered = self.clustering_obj_.predict(
280            self.clustering_scaler_.transform(X)
281        )
282
283        if self.cluster_encode == True:
284            return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
285                np.float16
286            )
287
288        return X_clustered.astype(np.float16)
289
290    def create_layer(self, scaled_X, W=None):
291        """Create hidden layer.
292
293        Parameters:
294
295            scaled_X: {array-like}, shape = [n_samples, n_features]
296                Training vectors, where n_samples is the number
297                of samples and n_features is the number of features
298
299            W: {array-like}, shape = [n_features, hidden_features]
300                if provided, constructs the hidden layer with W; otherwise computed internally
301
302        Returns:
303
304            Hidden layer matrix: {array-like}
305
306        """
307
308        n_features = scaled_X.shape[1]
309
310        # hash_sim = {
311        #         "sobol": generate_sobol,
312        #         "hammersley": generate_hammersley,
313        #         "uniform": generate_uniform,
314        #         "halton": generate_halton
315        #     }
316
317        if self.bias is False:  # no bias term in the hidden layer
318            if W is None:
319                if self.nodes_sim == "sobol":
320                    self.W_ = generate_sobol(
321                        n_dims=n_features,
322                        n_points=self.n_hidden_features,
323                        seed=self.seed,
324                    )
325                elif self.nodes_sim == "hammersley":
326                    self.W_ = generate_hammersley(
327                        n_dims=n_features,
328                        n_points=self.n_hidden_features,
329                        seed=self.seed,
330                    )
331                elif self.nodes_sim == "uniform":
332                    self.W_ = generate_uniform(
333                        n_dims=n_features,
334                        n_points=self.n_hidden_features,
335                        seed=self.seed,
336                    )
337                else:
338                    self.W_ = generate_halton(
339                        n_dims=n_features,
340                        n_points=self.n_hidden_features,
341                        seed=self.seed,
342                    )
343
344                # self.W_ = hash_sim[self.nodes_sim](
345                #             n_dims=n_features,
346                #             n_points=self.n_hidden_features,
347                #             seed=self.seed,
348                #         )
349
350                assert (
351                    scaled_X.shape[1] == self.W_.shape[0]
352                ), "check dimensions of covariates X and matrix W"
353
354                return mo.dropout(
355                    x=self.activation_func(
356                        mo.safe_sparse_dot(
357                            a=scaled_X, b=self.W_, backend=self.backend
358                        )
359                    ),
360                    drop_prob=self.dropout,
361                    seed=self.seed,
362                )
363
364            # W is not none
365            assert (
366                scaled_X.shape[1] == W.shape[0]
367            ), "check dimensions of covariates X and matrix W"
368
369            # self.W_ = W
370            return mo.dropout(
371                x=self.activation_func(
372                    mo.safe_sparse_dot(a=scaled_X, b=W, backend=self.backend)
373                ),
374                drop_prob=self.dropout,
375                seed=self.seed,
376            )
377
378        # with bias term in the hidden layer
379        if W is None:
380            n_features_1 = n_features + 1
381
382            if self.nodes_sim == "sobol":
383                self.W_ = generate_sobol(
384                    n_dims=n_features_1,
385                    n_points=self.n_hidden_features,
386                    seed=self.seed,
387                )
388            elif self.nodes_sim == "hammersley":
389                self.W_ = generate_hammersley(
390                    n_dims=n_features_1,
391                    n_points=self.n_hidden_features,
392                    seed=self.seed,
393                )
394            elif self.nodes_sim == "uniform":
395                self.W_ = generate_uniform(
396                    n_dims=n_features_1,
397                    n_points=self.n_hidden_features,
398                    seed=self.seed,
399                )
400            else:
401                self.W_ = generate_halton(
402                    n_dims=n_features_1,
403                    n_points=self.n_hidden_features,
404                    seed=self.seed,
405                )
406
407            # self.W_ = hash_sim[self.nodes_sim](
408            #         n_dims=n_features_1,
409            #         n_points=self.n_hidden_features,
410            #         seed=self.seed,
411            #     )
412
413            return mo.dropout(
414                x=self.activation_func(
415                    mo.safe_sparse_dot(
416                        a=mo.cbind(
417                            np.ones(scaled_X.shape[0]),
418                            scaled_X,
419                            backend=self.backend,
420                        ),
421                        b=self.W_,
422                        backend=self.backend,
423                    )
424                ),
425                drop_prob=self.dropout,
426                seed=self.seed,
427            )
428
429        # W is not None
430        # self.W_ = W
431        return mo.dropout(
432            x=self.activation_func(
433                mo.safe_sparse_dot(
434                    a=mo.cbind(
435                        np.ones(scaled_X.shape[0]),
436                        scaled_X,
437                        backend=self.backend,
438                    ),
439                    b=W,
440                    backend=self.backend,
441                )
442            ),
443            drop_prob=self.dropout,
444            seed=self.seed,
445        )
446
447    def cook_training_set(self, y=None, X=None, W=None, **kwargs):
448        """Create new hidden features for training set, with hidden layer, center the response.
449
450        Parameters:
451
452            y: array-like, shape = [n_samples]
453                Target values
454
455            X: {array-like}, shape = [n_samples, n_features]
456                Training vectors, where n_samples is the number
457                of samples and n_features is the number of features
458
459            W: {array-like}, shape = [n_features, hidden_features]
460                if provided, constructs the hidden layer via W
461
462        Returns:
463
464            (centered response, direct link + hidden layer matrix): {tuple}
465
466        """
467
468        # either X and y are stored or not
469        # assert ((y is None) & (X is None)) | ((y is not None) & (X is not None))
470        if self.n_hidden_features > 0:  # has a hidden layer
471            assert (
472                len(self.type_scaling) >= 2
473            ), "must have len(self.type_scaling) >= 2 when self.n_hidden_features > 0"
474
475        if X is None:
476            if self.col_sample == 1:
477                input_X = self.X_
478            else:
479                n_features = self.X_.shape[1]
480                new_n_features = int(np.ceil(n_features * self.col_sample))
481                assert (
482                    new_n_features >= 1
483                ), "check class attribute 'col_sample' and the number of covariates provided for X"
484                np.random.seed(self.seed)
485                index_col = np.random.choice(
486                    range(n_features), size=new_n_features, replace=False
487                )
488                self.index_col_ = index_col
489                input_X = self.X_[:, self.index_col_]
490
491        else:  # X is not None # keep X vs self.X_
492            if isinstance(X, pd.DataFrame):
493                X = copy.deepcopy(X.values.astype(float))
494
495            if self.col_sample == 1:
496                input_X = X
497            else:
498                n_features = X.shape[1]
499                new_n_features = int(np.ceil(n_features * self.col_sample))
500                assert (
501                    new_n_features >= 1
502                ), "check class attribute 'col_sample' and the number of covariates provided for X"
503                np.random.seed(self.seed)
504                index_col = np.random.choice(
505                    range(n_features), size=new_n_features, replace=False
506                )
507                self.index_col_ = index_col
508                input_X = X[:, self.index_col_]
509
510        if (
511            self.n_clusters <= 0
512        ):  # data without any clustering: self.n_clusters is None -----
513            if self.n_hidden_features > 0:  # with hidden layer
514                self.nn_scaler_, scaled_X = mo.scale_covariates(
515                    input_X, choice=self.type_scaling[1]
516                )
517                Phi_X = (
518                    self.create_layer(scaled_X)
519                    if W is None
520                    else self.create_layer(scaled_X, W=W)
521                )
522                Z = (
523                    mo.cbind(input_X, Phi_X, backend=self.backend)
524                    if self.direct_link is True
525                    else Phi_X
526                )
527                self.scaler_, scaled_Z = mo.scale_covariates(
528                    Z, choice=self.type_scaling[0]
529                )
530            else:  # no hidden layer
531                Z = input_X
532                self.scaler_, scaled_Z = mo.scale_covariates(
533                    Z, choice=self.type_scaling[0]
534                )
535        else:  # data with clustering: self.n_clusters is not None ----- # keep
536            augmented_X = mo.cbind(
537                input_X,
538                self.encode_clusters(input_X, **kwargs),
539                backend=self.backend,
540            )
541
542            if self.n_hidden_features > 0:  # with hidden layer
543                self.nn_scaler_, scaled_X = mo.scale_covariates(
544                    augmented_X, choice=self.type_scaling[1]
545                )
546                Phi_X = (
547                    self.create_layer(scaled_X)
548                    if W is None
549                    else self.create_layer(scaled_X, W=W)
550                )
551                Z = (
552                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
553                    if self.direct_link is True
554                    else Phi_X
555                )
556                self.scaler_, scaled_Z = mo.scale_covariates(
557                    Z, choice=self.type_scaling[0]
558                )
559            else:  # no hidden layer
560                Z = augmented_X
561                self.scaler_, scaled_Z = mo.scale_covariates(
562                    Z, choice=self.type_scaling[0]
563                )
564
565        # Returning model inputs -----
566        if mx.is_factor(y) is False:  # regression
567            # center y
568            if y is None:
569                self.y_mean_, centered_y = mo.center_response(self.y_)
570            else:
571                self.y_mean_, centered_y = mo.center_response(y)
572
573            # y is subsampled
574            if self.row_sample < 1:
575                n, p = Z.shape
576
577                self.subsampler_ = (
578                    SubSampler(
579                        y=self.y_, row_sample=self.row_sample, seed=self.seed
580                    )
581                    if y is None
582                    else SubSampler(
583                        y=y, row_sample=self.row_sample, seed=self.seed
584                    )
585                )
586
587                self.index_row_ = self.subsampler_.subsample()
588
589                n_row_sample = len(self.index_row_)
590                # regression
591                return (
592                    centered_y[self.index_row_].reshape(n_row_sample),
593                    self.scaler_.transform(
594                        Z[self.index_row_, :].reshape(n_row_sample, p)
595                    ),
596                )
597            # y is not subsampled
598            # regression
599            return (centered_y, self.scaler_.transform(Z))
600
601        # classification
602        # y is subsampled
603        if self.row_sample < 1:
604            n, p = Z.shape
605
606            self.subsampler_ = (
607                SubSampler(
608                    y=self.y_, row_sample=self.row_sample, seed=self.seed
609                )
610                if y is None
611                else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed)
612            )
613
614            self.index_row_ = self.subsampler_.subsample()
615
616            n_row_sample = len(self.index_row_)
617            # classification
618            return (
619                y[self.index_row_].reshape(n_row_sample),
620                self.scaler_.transform(
621                    Z[self.index_row_, :].reshape(n_row_sample, p)
622                ),
623            )
624        # y is not subsampled
625        # classification
626        return (y, self.scaler_.transform(Z))
627
628    def cook_test_set(self, X, **kwargs):
629        """Transform data from test set, with hidden layer.
630
631        Parameters:
632
633            X: {array-like}, shape = [n_samples, n_features]
634                Training vectors, where n_samples is the number
635                of samples and n_features is the number of features
636
637            **kwargs: additional parameters to be passed to self.encode_cluster
638
639        Returns:
640
641            Transformed test set : {array-like}
642        """
643
644        if isinstance(X, pd.DataFrame):
645            X = copy.deepcopy(X.values.astype(float))
646
647        if (
648            self.n_clusters == 0
649        ):  # data without clustering: self.n_clusters is None -----
650            if self.n_hidden_features > 0:
651                # if hidden layer
652                scaled_X = (
653                    self.nn_scaler_.transform(X)
654                    if (self.col_sample == 1)
655                    else self.nn_scaler_.transform(X[:, self.index_col_])
656                )
657                Phi_X = self.create_layer(scaled_X, self.W_)
658                if self.direct_link == True:
659                    return self.scaler_.transform(
660                        mo.cbind(scaled_X, Phi_X, backend=self.backend)
661                    )
662                # when self.direct_link == False
663                return self.scaler_.transform(Phi_X)
664            # if no hidden layer # self.n_hidden_features == 0
665            return self.scaler_.transform(X)
666
667        # data with clustering: self.n_clusters > 0 -----
668        if self.col_sample == 1:
669            predicted_clusters = self.encode_clusters(
670                X=X, predict=True, **kwargs
671            )
672            augmented_X = mo.cbind(X, predicted_clusters, backend=self.backend)
673        else:
674            predicted_clusters = self.encode_clusters(
675                X=X[:, self.index_col_], predict=True, **kwargs
676            )
677            augmented_X = mo.cbind(
678                X[:, self.index_col_], predicted_clusters, backend=self.backend
679            )
680
681        if self.n_hidden_features > 0:  # if hidden layer
682            scaled_X = self.nn_scaler_.transform(augmented_X)
683            Phi_X = self.create_layer(scaled_X, self.W_)
684            if self.direct_link == True:
685                return self.scaler_.transform(
686                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
687                )
688            return self.scaler_.transform(Phi_X)
689
690        # if no hidden layer
691        return self.scaler_.transform(augmented_X)

Base model from which all the other classes inherit.

This class contains the most important data preprocessing/feature engineering methods.

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or
    not (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original features are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for type_clust='kmeans' or type_clust='gmm'
    clustering (could be 0: no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot);
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax') or robust scaling ('robust')

col_sample: float
    percentage of features randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform', clustering and dropout

backend: str
    "cpu" or "gpu" or "tpu"
Base( n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
 96    def __init__(
 97        self,
 98        n_hidden_features=5,
 99        activation_name="relu",
100        a=0.01,
101        nodes_sim="sobol",
102        bias=True,
103        dropout=0,
104        direct_link=True,
105        n_clusters=2,
106        cluster_encode=True,
107        type_clust="kmeans",
108        type_scaling=("std", "std", "std"),
109        col_sample=1,
110        row_sample=1,
111        seed=123,
112        backend="cpu",
113    ):
114        # input checks -----
115
116        sys_platform = platform.system()
117
118        if (sys_platform == "Windows") and (backend in ("gpu", "tpu")):
119            warnings.warn(
120                "No GPU/TPU computing on Windows yet, backend set to 'cpu'"
121            )
122            backend = "cpu"
123
124        assert activation_name in (
125            "relu",
126            "tanh",
127            "sigmoid",
128            "prelu",
129            "elu",
130        ), "'activation_name' must be in ('relu', 'tanh', 'sigmoid','prelu', 'elu')"
131
132        assert nodes_sim in (
133            "sobol",
134            "hammersley",
135            "uniform",
136            "halton",
137        ), "'nodes_sim' must be in ('sobol', 'hammersley', 'uniform', 'halton')"
138
139        assert type_clust in (
140            "kmeans",
141            "gmm",
142        ), "'type_clust' must be in ('kmeans', 'gmm')"
143
144        assert (len(type_scaling) == 3) & all(
145            type_scaling[i] in ("minmax", "std", "robust")
146            for i in range(len(type_scaling))
147        ), "'type_scaling' must have length 3, and available scaling methods are 'minmax' scaling, standardization ('std') and robust scaling ('robust')"
148
149        assert (col_sample >= 0) & (
150            col_sample <= 1
151        ), "'col_sample' must be comprised between 0 and 1 (both included)"
152
153        assert backend in (
154            "cpu",
155            "gpu",
156            "tpu",
157        ), "must have 'backend' in ('cpu', 'gpu', 'tpu')"
158
159        self.n_hidden_features = n_hidden_features
160        self.activation_name = activation_name
161        self.a = a
162        self.nodes_sim = nodes_sim
163        self.bias = bias
164        self.seed = seed
165        self.backend = backend
166        self.dropout = dropout
167        self.direct_link = direct_link
168        self.cluster_encode = cluster_encode
169        self.type_clust = type_clust
170        self.type_scaling = type_scaling
171        self.col_sample = col_sample
172        self.row_sample = row_sample
173        self.n_clusters = n_clusters
174        self.subsampler_ = None
175        self.index_col_ = None
176        self.index_row_ = True
177        self.clustering_obj_ = None
178        self.clustering_scaler_ = None
179        self.nn_scaler_ = None
180        self.scaler_ = None
181        self.encoder_ = None
182        self.W_ = None
183        self.X_ = None
184        self.y_ = None
185        self.y_mean_ = None
186        self.beta_ = None
187
188        # activation function -----
189        if sys_platform in ("Linux", "Darwin"):
190            activation_options = {
191                "relu": ac.relu if (self.backend == "cpu") else jnn.relu,
192                "tanh": np.tanh if (self.backend == "cpu") else jnp.tanh,
193                "sigmoid": (
194                    ac.sigmoid if (self.backend == "cpu") else jnn.sigmoid
195                ),
196                "prelu": partial(ac.prelu, a=a),
197                "elu": (
198                    partial(ac.elu, a=a)
199                    if (self.backend == "cpu")
200                    else partial(jnn.elu, a=a)
201                ),
202            }
203        else:  # on Windows currently, no JAX
204            activation_options = {
205                "relu": (
206                    ac.relu if (self.backend == "cpu") else NotImplementedError
207                ),
208                "tanh": (
209                    np.tanh if (self.backend == "cpu") else NotImplementedError
210                ),
211                "sigmoid": (
212                    ac.sigmoid
213                    if (self.backend == "cpu")
214                    else NotImplementedError
215                ),
216                "prelu": partial(ac.prelu, a=a),
217                "elu": (
218                    partial(ac.elu, a=a)
219                    if (self.backend == "cpu")
220                    else NotImplementedError
221                ),
222            }
223        self.activation_func = activation_options[activation_name]
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
def encode_clusters(self, X=None, predict=False, **kwargs):
227    def encode_clusters(self, X=None, predict=False, **kwargs):  #
228        """Create new covariates with kmeans or GMM clustering
229
230        Parameters:
231
232            X: {array-like}, shape = [n_samples, n_features]
233                Training vectors, where n_samples is the number
234                of samples and n_features is the number of features.
235
236            predict: boolean
237                is False on training set and True on test set
238
239            **kwargs:
240                additional parameters to be passed to the
241                clustering method
242
243        Returns:
244
245            Clusters' matrix, one-hot encoded: {array-like}
246
247        """
248
249        np.random.seed(self.seed)
250
251        if X is None:
252            X = self.X_
253
254        if isinstance(X, pd.DataFrame):
255            X = copy.deepcopy(X.values.astype(float))
256
257        if predict is False:  # encode training set
258            # scale input data before clustering
259            self.clustering_scaler_, scaled_X = mo.scale_covariates(
260                X, choice=self.type_scaling[2]
261            )
262
263            self.clustering_obj_, X_clustered = mo.cluster_covariates(
264                scaled_X,
265                self.n_clusters,
266                self.seed,
267                type_clust=self.type_clust,
268                **kwargs
269            )
270
271            if self.cluster_encode == True:
272                return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
273                    np.float16
274                )
275
276            return X_clustered.astype(np.float16)
277
278        # if predict == True, encode test set
279        X_clustered = self.clustering_obj_.predict(
280            self.clustering_scaler_.transform(X)
281        )
282
283        if self.cluster_encode == True:
284            return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
285                np.float16
286            )
287
288        return X_clustered.astype(np.float16)

Create new covariates with kmeans or GMM clustering

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

predict: boolean
    is False on training set and True on test set

**kwargs:
    additional parameters to be passed to the
    clustering method

Returns:

Clusters' matrix, one-hot encoded: {array-like}
def create_layer(self, scaled_X, W=None):
290    def create_layer(self, scaled_X, W=None):
291        """Create hidden layer.
292
293        Parameters:
294
295            scaled_X: {array-like}, shape = [n_samples, n_features]
296                Training vectors, where n_samples is the number
297                of samples and n_features is the number of features
298
299            W: {array-like}, shape = [n_features, hidden_features]
300                if provided, constructs the hidden layer with W; otherwise computed internally
301
302        Returns:
303
304            Hidden layer matrix: {array-like}
305
306        """
307
308        n_features = scaled_X.shape[1]
309
310        # hash_sim = {
311        #         "sobol": generate_sobol,
312        #         "hammersley": generate_hammersley,
313        #         "uniform": generate_uniform,
314        #         "halton": generate_halton
315        #     }
316
317        if self.bias is False:  # no bias term in the hidden layer
318            if W is None:
319                if self.nodes_sim == "sobol":
320                    self.W_ = generate_sobol(
321                        n_dims=n_features,
322                        n_points=self.n_hidden_features,
323                        seed=self.seed,
324                    )
325                elif self.nodes_sim == "hammersley":
326                    self.W_ = generate_hammersley(
327                        n_dims=n_features,
328                        n_points=self.n_hidden_features,
329                        seed=self.seed,
330                    )
331                elif self.nodes_sim == "uniform":
332                    self.W_ = generate_uniform(
333                        n_dims=n_features,
334                        n_points=self.n_hidden_features,
335                        seed=self.seed,
336                    )
337                else:
338                    self.W_ = generate_halton(
339                        n_dims=n_features,
340                        n_points=self.n_hidden_features,
341                        seed=self.seed,
342                    )
343
344                # self.W_ = hash_sim[self.nodes_sim](
345                #             n_dims=n_features,
346                #             n_points=self.n_hidden_features,
347                #             seed=self.seed,
348                #         )
349
350                assert (
351                    scaled_X.shape[1] == self.W_.shape[0]
352                ), "check dimensions of covariates X and matrix W"
353
354                return mo.dropout(
355                    x=self.activation_func(
356                        mo.safe_sparse_dot(
357                            a=scaled_X, b=self.W_, backend=self.backend
358                        )
359                    ),
360                    drop_prob=self.dropout,
361                    seed=self.seed,
362                )
363
364            # W is not none
365            assert (
366                scaled_X.shape[1] == W.shape[0]
367            ), "check dimensions of covariates X and matrix W"
368
369            # self.W_ = W
370            return mo.dropout(
371                x=self.activation_func(
372                    mo.safe_sparse_dot(a=scaled_X, b=W, backend=self.backend)
373                ),
374                drop_prob=self.dropout,
375                seed=self.seed,
376            )
377
378        # with bias term in the hidden layer
379        if W is None:
380            n_features_1 = n_features + 1
381
382            if self.nodes_sim == "sobol":
383                self.W_ = generate_sobol(
384                    n_dims=n_features_1,
385                    n_points=self.n_hidden_features,
386                    seed=self.seed,
387                )
388            elif self.nodes_sim == "hammersley":
389                self.W_ = generate_hammersley(
390                    n_dims=n_features_1,
391                    n_points=self.n_hidden_features,
392                    seed=self.seed,
393                )
394            elif self.nodes_sim == "uniform":
395                self.W_ = generate_uniform(
396                    n_dims=n_features_1,
397                    n_points=self.n_hidden_features,
398                    seed=self.seed,
399                )
400            else:
401                self.W_ = generate_halton(
402                    n_dims=n_features_1,
403                    n_points=self.n_hidden_features,
404                    seed=self.seed,
405                )
406
407            # self.W_ = hash_sim[self.nodes_sim](
408            #         n_dims=n_features_1,
409            #         n_points=self.n_hidden_features,
410            #         seed=self.seed,
411            #     )
412
413            return mo.dropout(
414                x=self.activation_func(
415                    mo.safe_sparse_dot(
416                        a=mo.cbind(
417                            np.ones(scaled_X.shape[0]),
418                            scaled_X,
419                            backend=self.backend,
420                        ),
421                        b=self.W_,
422                        backend=self.backend,
423                    )
424                ),
425                drop_prob=self.dropout,
426                seed=self.seed,
427            )
428
429        # W is not None
430        # self.W_ = W
431        return mo.dropout(
432            x=self.activation_func(
433                mo.safe_sparse_dot(
434                    a=mo.cbind(
435                        np.ones(scaled_X.shape[0]),
436                        scaled_X,
437                        backend=self.backend,
438                    ),
439                    b=W,
440                    backend=self.backend,
441                )
442            ),
443            drop_prob=self.dropout,
444            seed=self.seed,
445        )

Create hidden layer.

Parameters:

scaled_X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

W: {array-like}, shape = [n_features, hidden_features]
    if provided, constructs the hidden layer with W; otherwise computed internally

Returns:

Hidden layer matrix: {array-like}
def cook_training_set(self, y=None, X=None, W=None, **kwargs):
447    def cook_training_set(self, y=None, X=None, W=None, **kwargs):
448        """Create new hidden features for training set, with hidden layer, center the response.
449
450        Parameters:
451
452            y: array-like, shape = [n_samples]
453                Target values
454
455            X: {array-like}, shape = [n_samples, n_features]
456                Training vectors, where n_samples is the number
457                of samples and n_features is the number of features
458
459            W: {array-like}, shape = [n_features, hidden_features]
460                if provided, constructs the hidden layer via W
461
462        Returns:
463
464            (centered response, direct link + hidden layer matrix): {tuple}
465
466        """
467
468        # either X and y are stored or not
469        # assert ((y is None) & (X is None)) | ((y is not None) & (X is not None))
470        if self.n_hidden_features > 0:  # has a hidden layer
471            assert (
472                len(self.type_scaling) >= 2
473            ), "must have len(self.type_scaling) >= 2 when self.n_hidden_features > 0"
474
475        if X is None:
476            if self.col_sample == 1:
477                input_X = self.X_
478            else:
479                n_features = self.X_.shape[1]
480                new_n_features = int(np.ceil(n_features * self.col_sample))
481                assert (
482                    new_n_features >= 1
483                ), "check class attribute 'col_sample' and the number of covariates provided for X"
484                np.random.seed(self.seed)
485                index_col = np.random.choice(
486                    range(n_features), size=new_n_features, replace=False
487                )
488                self.index_col_ = index_col
489                input_X = self.X_[:, self.index_col_]
490
491        else:  # X is not None # keep X vs self.X_
492            if isinstance(X, pd.DataFrame):
493                X = copy.deepcopy(X.values.astype(float))
494
495            if self.col_sample == 1:
496                input_X = X
497            else:
498                n_features = X.shape[1]
499                new_n_features = int(np.ceil(n_features * self.col_sample))
500                assert (
501                    new_n_features >= 1
502                ), "check class attribute 'col_sample' and the number of covariates provided for X"
503                np.random.seed(self.seed)
504                index_col = np.random.choice(
505                    range(n_features), size=new_n_features, replace=False
506                )
507                self.index_col_ = index_col
508                input_X = X[:, self.index_col_]
509
510        if (
511            self.n_clusters <= 0
512        ):  # data without any clustering: self.n_clusters is None -----
513            if self.n_hidden_features > 0:  # with hidden layer
514                self.nn_scaler_, scaled_X = mo.scale_covariates(
515                    input_X, choice=self.type_scaling[1]
516                )
517                Phi_X = (
518                    self.create_layer(scaled_X)
519                    if W is None
520                    else self.create_layer(scaled_X, W=W)
521                )
522                Z = (
523                    mo.cbind(input_X, Phi_X, backend=self.backend)
524                    if self.direct_link is True
525                    else Phi_X
526                )
527                self.scaler_, scaled_Z = mo.scale_covariates(
528                    Z, choice=self.type_scaling[0]
529                )
530            else:  # no hidden layer
531                Z = input_X
532                self.scaler_, scaled_Z = mo.scale_covariates(
533                    Z, choice=self.type_scaling[0]
534                )
535        else:  # data with clustering: self.n_clusters is not None ----- # keep
536            augmented_X = mo.cbind(
537                input_X,
538                self.encode_clusters(input_X, **kwargs),
539                backend=self.backend,
540            )
541
542            if self.n_hidden_features > 0:  # with hidden layer
543                self.nn_scaler_, scaled_X = mo.scale_covariates(
544                    augmented_X, choice=self.type_scaling[1]
545                )
546                Phi_X = (
547                    self.create_layer(scaled_X)
548                    if W is None
549                    else self.create_layer(scaled_X, W=W)
550                )
551                Z = (
552                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
553                    if self.direct_link is True
554                    else Phi_X
555                )
556                self.scaler_, scaled_Z = mo.scale_covariates(
557                    Z, choice=self.type_scaling[0]
558                )
559            else:  # no hidden layer
560                Z = augmented_X
561                self.scaler_, scaled_Z = mo.scale_covariates(
562                    Z, choice=self.type_scaling[0]
563                )
564
565        # Returning model inputs -----
566        if mx.is_factor(y) is False:  # regression
567            # center y
568            if y is None:
569                self.y_mean_, centered_y = mo.center_response(self.y_)
570            else:
571                self.y_mean_, centered_y = mo.center_response(y)
572
573            # y is subsampled
574            if self.row_sample < 1:
575                n, p = Z.shape
576
577                self.subsampler_ = (
578                    SubSampler(
579                        y=self.y_, row_sample=self.row_sample, seed=self.seed
580                    )
581                    if y is None
582                    else SubSampler(
583                        y=y, row_sample=self.row_sample, seed=self.seed
584                    )
585                )
586
587                self.index_row_ = self.subsampler_.subsample()
588
589                n_row_sample = len(self.index_row_)
590                # regression
591                return (
592                    centered_y[self.index_row_].reshape(n_row_sample),
593                    self.scaler_.transform(
594                        Z[self.index_row_, :].reshape(n_row_sample, p)
595                    ),
596                )
597            # y is not subsampled
598            # regression
599            return (centered_y, self.scaler_.transform(Z))
600
601        # classification
602        # y is subsampled
603        if self.row_sample < 1:
604            n, p = Z.shape
605
606            self.subsampler_ = (
607                SubSampler(
608                    y=self.y_, row_sample=self.row_sample, seed=self.seed
609                )
610                if y is None
611                else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed)
612            )
613
614            self.index_row_ = self.subsampler_.subsample()
615
616            n_row_sample = len(self.index_row_)
617            # classification
618            return (
619                y[self.index_row_].reshape(n_row_sample),
620                self.scaler_.transform(
621                    Z[self.index_row_, :].reshape(n_row_sample, p)
622                ),
623            )
624        # y is not subsampled
625        # classification
626        return (y, self.scaler_.transform(Z))

Create new hidden features for training set, with hidden layer, center the response.

Parameters:

y: array-like, shape = [n_samples]
    Target values

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

W: {array-like}, shape = [n_features, hidden_features]
    if provided, constructs the hidden layer via W

Returns:

(centered response, direct link + hidden layer matrix): {tuple}
def cook_test_set(self, X, **kwargs):
628    def cook_test_set(self, X, **kwargs):
629        """Transform data from test set, with hidden layer.
630
631        Parameters:
632
633            X: {array-like}, shape = [n_samples, n_features]
634                Training vectors, where n_samples is the number
635                of samples and n_features is the number of features
636
637            **kwargs: additional parameters to be passed to self.encode_cluster
638
639        Returns:
640
641            Transformed test set : {array-like}
642        """
643
644        if isinstance(X, pd.DataFrame):
645            X = copy.deepcopy(X.values.astype(float))
646
647        if (
648            self.n_clusters == 0
649        ):  # data without clustering: self.n_clusters is None -----
650            if self.n_hidden_features > 0:
651                # if hidden layer
652                scaled_X = (
653                    self.nn_scaler_.transform(X)
654                    if (self.col_sample == 1)
655                    else self.nn_scaler_.transform(X[:, self.index_col_])
656                )
657                Phi_X = self.create_layer(scaled_X, self.W_)
658                if self.direct_link == True:
659                    return self.scaler_.transform(
660                        mo.cbind(scaled_X, Phi_X, backend=self.backend)
661                    )
662                # when self.direct_link == False
663                return self.scaler_.transform(Phi_X)
664            # if no hidden layer # self.n_hidden_features == 0
665            return self.scaler_.transform(X)
666
667        # data with clustering: self.n_clusters > 0 -----
668        if self.col_sample == 1:
669            predicted_clusters = self.encode_clusters(
670                X=X, predict=True, **kwargs
671            )
672            augmented_X = mo.cbind(X, predicted_clusters, backend=self.backend)
673        else:
674            predicted_clusters = self.encode_clusters(
675                X=X[:, self.index_col_], predict=True, **kwargs
676            )
677            augmented_X = mo.cbind(
678                X[:, self.index_col_], predicted_clusters, backend=self.backend
679            )
680
681        if self.n_hidden_features > 0:  # if hidden layer
682            scaled_X = self.nn_scaler_.transform(augmented_X)
683            Phi_X = self.create_layer(scaled_X, self.W_)
684            if self.direct_link == True:
685                return self.scaler_.transform(
686                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
687                )
688            return self.scaler_.transform(Phi_X)
689
690        # if no hidden layer
691        return self.scaler_.transform(augmented_X)

Transform data from test set, with hidden layer.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

**kwargs: additional parameters to be passed to self.encode_cluster

Returns:

Transformed test set : {array-like}
Inherited Members
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
class BaseRegressor(nnetsauce.Base, sklearn.base.RegressorMixin):
 15class BaseRegressor(Base, RegressorMixin):
 16    """Random Vector Functional Link Network regression without shrinkage
 17
 18    Parameters:
 19
 20        n_hidden_features: int
 21            number of nodes in the hidden layer
 22
 23        activation_name: str
 24            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 25
 26        a: float
 27            hyperparameter for 'prelu' or 'elu' activation function
 28
 29        nodes_sim: str
 30            type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
 31            'uniform'
 32
 33        bias: boolean
 34            indicates if the hidden layer contains a bias term (True) or
 35            not (False)
 36
 37        dropout: float
 38            regularization parameter; (random) percentage of nodes dropped out
 39            of the training
 40
 41        direct_link: boolean
 42            indicates if the original features are included (True) in model's
 43            fitting or not (False)
 44
 45        n_clusters: int
 46            number of clusters for type_clust='kmeans' or type_clust='gmm'
 47            clustering (could be 0: no clustering)
 48
 49        cluster_encode: bool
 50            defines how the variable containing clusters is treated (default is one-hot);
 51            if `False`, then labels are used, without one-hot encoding
 52
 53        type_clust: str
 54            type of clustering method: currently k-means ('kmeans') or Gaussian
 55            Mixture Model ('gmm')
 56
 57        type_scaling: a tuple of 3 strings
 58            scaling methods for inputs, hidden layer, and clustering respectively
 59            (and when relevant).
 60            Currently available: standardization ('std') or MinMax scaling ('minmax')
 61
 62        col_sample: float
 63            percentage of features randomly chosen for training
 64
 65        row_sample: float
 66            percentage of rows chosen for training, by stratified bootstrapping
 67
 68        seed: int
 69            reproducibility seed for nodes_sim=='uniform', clustering and dropout
 70
 71        backend: str
 72            "cpu" or "gpu" or "tpu"
 73
 74    Attributes:
 75
 76        beta_: vector
 77            regression coefficients
 78
 79        GCV_: float
 80            Generalized Cross-Validation error
 81
 82    """
 83
 84    # construct the object -----
 85
 86    def __init__(
 87        self,
 88        n_hidden_features=5,
 89        activation_name="relu",
 90        a=0.01,
 91        nodes_sim="sobol",
 92        bias=True,
 93        dropout=0,
 94        direct_link=True,
 95        n_clusters=2,
 96        cluster_encode=True,
 97        type_clust="kmeans",
 98        type_scaling=("std", "std", "std"),
 99        col_sample=1,
100        row_sample=1,
101        seed=123,
102        backend="cpu",
103    ):
104        super().__init__(
105            n_hidden_features=n_hidden_features,
106            activation_name=activation_name,
107            a=a,
108            nodes_sim=nodes_sim,
109            bias=bias,
110            dropout=dropout,
111            direct_link=direct_link,
112            n_clusters=n_clusters,
113            cluster_encode=cluster_encode,
114            type_clust=type_clust,
115            type_scaling=type_scaling,
116            col_sample=col_sample,
117            row_sample=row_sample,
118            seed=seed,
119            backend=backend,
120        )
121
122    def fit(self, X, y, **kwargs):
123        """Fit BaseRegressor to training data (X, y)
124
125        Parameters:
126
127            X: {array-like}, shape = [n_samples, n_features]
128                Training vectors, where n_samples is the number
129                of samples and n_features is the number of features
130
131            y: array-like, shape = [n_samples]
132                Target values
133
134            **kwargs: additional parameters to be passed to self.cook_training_set
135
136        Returns:
137
138            self: object
139        """
140
141        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
142
143        fit_obj = lmf.beta_Sigma_hat(
144            X=scaled_Z, y=centered_y, backend=self.backend
145        )
146
147        self.beta_ = fit_obj["beta_hat"]
148
149        self.GCV_ = fit_obj["GCV"]
150
151        return self
152
153    def predict(self, X, **kwargs):
154        """Predict test data X.
155
156        Parameters:
157
158            X: {array-like}, shape = [n_samples, n_features]
159                Training vectors, where n_samples is the number
160                of samples and n_features is the number of features
161
162            **kwargs: additional parameters to be passed to self.cook_test_set
163
164        Returns:
165
166            model predictions: {array-like}
167        """
168
169        if len(X.shape) == 1:
170            n_features = X.shape[0]
171            new_X = mo.rbind(
172                X.reshape(1, n_features),
173                np.ones(n_features).reshape(1, n_features),
174            )
175
176            return (
177                self.y_mean_
178                + mo.safe_sparse_dot(
179                    a=self.cook_test_set(new_X, **kwargs),
180                    b=self.beta_,
181                    backend=self.backend,
182                )
183            )[0]
184
185        return self.y_mean_ + mo.safe_sparse_dot(
186            a=self.cook_test_set(X, **kwargs),
187            b=self.beta_,
188            backend=self.backend,
189        )
190
191    def score(self, X, y, scoring=None, **kwargs):
192        """Score the model on test set features X and response y.
193
194        Parameters:
195
196            X: {array-like}, shape = [n_samples, n_features]
197                Training vectors, where n_samples is the number
198                of samples and n_features is the number of features
199
200            y: array-like, shape = [n_samples]
201                Target values
202
203            scoring: str
204                must be in ('explained_variance', 'neg_mean_absolute_error',
205                            'neg_mean_squared_error', 'neg_mean_squared_log_error',
206                            'neg_median_absolute_error', 'r2')
207
208            **kwargs: additional parameters to be passed to scoring functions
209
210        Returns:
211
212        model scores: {array-like}
213
214        """
215
216        preds = self.predict(X)
217
218        if type(preds) == tuple:  # if there are std. devs in the predictions
219            preds = preds[0]
220
221        if scoring is None:
222            scoring = "neg_mean_squared_error"
223
224        # check inputs
225        assert scoring in (
226            "explained_variance",
227            "neg_mean_absolute_error",
228            "neg_mean_squared_error",
229            "neg_mean_squared_log_error",
230            "neg_median_absolute_error",
231            "r2",
232        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
233                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
234                           'neg_median_absolute_error', 'r2')"
235
236        scoring_options = {
237            "explained_variance": skm.explained_variance_score,
238            "neg_mean_absolute_error": skm.median_absolute_error,
239            "neg_mean_squared_error": skm.mean_squared_error,
240            "neg_mean_squared_log_error": skm.mean_squared_log_error,
241            "neg_median_absolute_error": skm.median_absolute_error,
242            "r2": skm.r2_score,
243        }
244
245        return scoring_options[scoring](y, preds, **kwargs)

Random Vector Functional Link Network regression without shrinkage

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or
    not (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original features are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for type_clust='kmeans' or type_clust='gmm'
    clustering (could be 0: no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot);
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of features randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform', clustering and dropout

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

beta_: vector
    regression coefficients

GCV_: float
    Generalized Cross-Validation error
BaseRegressor( n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
 86    def __init__(
 87        self,
 88        n_hidden_features=5,
 89        activation_name="relu",
 90        a=0.01,
 91        nodes_sim="sobol",
 92        bias=True,
 93        dropout=0,
 94        direct_link=True,
 95        n_clusters=2,
 96        cluster_encode=True,
 97        type_clust="kmeans",
 98        type_scaling=("std", "std", "std"),
 99        col_sample=1,
100        row_sample=1,
101        seed=123,
102        backend="cpu",
103    ):
104        super().__init__(
105            n_hidden_features=n_hidden_features,
106            activation_name=activation_name,
107            a=a,
108            nodes_sim=nodes_sim,
109            bias=bias,
110            dropout=dropout,
111            direct_link=direct_link,
112            n_clusters=n_clusters,
113            cluster_encode=cluster_encode,
114            type_clust=type_clust,
115            type_scaling=type_scaling,
116            col_sample=col_sample,
117            row_sample=row_sample,
118            seed=seed,
119            backend=backend,
120        )
def fit(self, X, y, **kwargs):
122    def fit(self, X, y, **kwargs):
123        """Fit BaseRegressor to training data (X, y)
124
125        Parameters:
126
127            X: {array-like}, shape = [n_samples, n_features]
128                Training vectors, where n_samples is the number
129                of samples and n_features is the number of features
130
131            y: array-like, shape = [n_samples]
132                Target values
133
134            **kwargs: additional parameters to be passed to self.cook_training_set
135
136        Returns:
137
138            self: object
139        """
140
141        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
142
143        fit_obj = lmf.beta_Sigma_hat(
144            X=scaled_Z, y=centered_y, backend=self.backend
145        )
146
147        self.beta_ = fit_obj["beta_hat"]
148
149        self.GCV_ = fit_obj["GCV"]
150
151        return self

Fit BaseRegressor to training data (X, y)

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

**kwargs: additional parameters to be passed to self.cook_training_set

Returns:

self: object
def predict(self, X, **kwargs):
153    def predict(self, X, **kwargs):
154        """Predict test data X.
155
156        Parameters:
157
158            X: {array-like}, shape = [n_samples, n_features]
159                Training vectors, where n_samples is the number
160                of samples and n_features is the number of features
161
162            **kwargs: additional parameters to be passed to self.cook_test_set
163
164        Returns:
165
166            model predictions: {array-like}
167        """
168
169        if len(X.shape) == 1:
170            n_features = X.shape[0]
171            new_X = mo.rbind(
172                X.reshape(1, n_features),
173                np.ones(n_features).reshape(1, n_features),
174            )
175
176            return (
177                self.y_mean_
178                + mo.safe_sparse_dot(
179                    a=self.cook_test_set(new_X, **kwargs),
180                    b=self.beta_,
181                    backend=self.backend,
182                )
183            )[0]
184
185        return self.y_mean_ + mo.safe_sparse_dot(
186            a=self.cook_test_set(X, **kwargs),
187            b=self.beta_,
188            backend=self.backend,
189        )

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

**kwargs: additional parameters to be passed to self.cook_test_set

Returns:

model predictions: {array-like}
def score(self, X, y, scoring=None, **kwargs):
191    def score(self, X, y, scoring=None, **kwargs):
192        """Score the model on test set features X and response y.
193
194        Parameters:
195
196            X: {array-like}, shape = [n_samples, n_features]
197                Training vectors, where n_samples is the number
198                of samples and n_features is the number of features
199
200            y: array-like, shape = [n_samples]
201                Target values
202
203            scoring: str
204                must be in ('explained_variance', 'neg_mean_absolute_error',
205                            'neg_mean_squared_error', 'neg_mean_squared_log_error',
206                            'neg_median_absolute_error', 'r2')
207
208            **kwargs: additional parameters to be passed to scoring functions
209
210        Returns:
211
212        model scores: {array-like}
213
214        """
215
216        preds = self.predict(X)
217
218        if type(preds) == tuple:  # if there are std. devs in the predictions
219            preds = preds[0]
220
221        if scoring is None:
222            scoring = "neg_mean_squared_error"
223
224        # check inputs
225        assert scoring in (
226            "explained_variance",
227            "neg_mean_absolute_error",
228            "neg_mean_squared_error",
229            "neg_mean_squared_log_error",
230            "neg_median_absolute_error",
231            "r2",
232        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
233                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
234                           'neg_median_absolute_error', 'r2')"
235
236        scoring_options = {
237            "explained_variance": skm.explained_variance_score,
238            "neg_mean_absolute_error": skm.median_absolute_error,
239            "neg_mean_squared_error": skm.mean_squared_error,
240            "neg_mean_squared_log_error": skm.mean_squared_log_error,
241            "neg_median_absolute_error": skm.median_absolute_error,
242            "r2": skm.r2_score,
243        }
244
245        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',
                'neg_mean_squared_error', 'neg_mean_squared_log_error',
                'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class BayesianRVFLRegressor(nnetsauce.Base, sklearn.base.RegressorMixin):
 15class BayesianRVFLRegressor(Base, RegressorMixin):
 16    """Bayesian Random Vector Functional Link Network regression with one prior
 17
 18    Parameters:
 19
 20        n_hidden_features: int
 21            number of nodes in the hidden layer
 22
 23        activation_name: str
 24            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 25
 26        a: float
 27            hyperparameter for 'prelu' or 'elu' activation function
 28
 29        nodes_sim: str
 30            type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform'
 31
 32        bias: boolean
 33            indicates if the hidden layer contains a bias term (True) or not (False)
 34
 35        dropout: float
 36            regularization parameter; (random) percentage of nodes dropped out
 37            of the training
 38
 39        direct_link: boolean
 40            indicates if the original features are included (True) in model''s fitting or not (False)
 41
 42        n_clusters: int
 43            number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering)
 44
 45        cluster_encode: bool
 46            defines how the variable containing clusters is treated (default is one-hot)
 47            if `False`, then labels are used, without one-hot encoding
 48
 49        type_clust: str
 50            type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm')
 51
 52        type_scaling: a tuple of 3 strings
 53            scaling methods for inputs, hidden layer, and clustering respectively
 54            (and when relevant).
 55            Currently available: standardization ('std') or MinMax scaling ('minmax')
 56
 57        seed: int
 58            reproducibility seed for nodes_sim=='uniform'
 59
 60        s: float
 61            std. dev. of regression parameters in Bayesian Ridge Regression
 62
 63        sigma: float
 64            std. dev. of residuals in Bayesian Ridge Regression
 65
 66        return_std: boolean
 67            if True, uncertainty around predictions is evaluated
 68
 69        backend: str
 70            "cpu" or "gpu" or "tpu"
 71
 72    Attributes:
 73
 74        beta_: array-like
 75            regression''s coefficients
 76
 77        Sigma_: array-like
 78            covariance of the distribution of fitted parameters
 79
 80        GCV_: float
 81            Generalized cross-validation error
 82
 83        y_mean_: float
 84            average response
 85
 86    Examples:
 87
 88    ```python
 89    TBD
 90    ```
 91
 92    """
 93
 94    # construct the object -----
 95
 96    def __init__(
 97        self,
 98        n_hidden_features=5,
 99        activation_name="relu",
100        a=0.01,
101        nodes_sim="sobol",
102        bias=True,
103        dropout=0,
104        direct_link=True,
105        n_clusters=2,
106        cluster_encode=True,
107        type_clust="kmeans",
108        type_scaling=("std", "std", "std"),
109        seed=123,
110        s=0.1,
111        sigma=0.05,
112        return_std=True,
113        backend="cpu",
114    ):
115        super().__init__(
116            n_hidden_features=n_hidden_features,
117            activation_name=activation_name,
118            a=a,
119            nodes_sim=nodes_sim,
120            bias=bias,
121            dropout=dropout,
122            direct_link=direct_link,
123            n_clusters=n_clusters,
124            cluster_encode=cluster_encode,
125            type_clust=type_clust,
126            type_scaling=type_scaling,
127            seed=seed,
128            backend=backend,
129        )
130        self.s = s
131        self.sigma = sigma
132        self.beta_ = None
133        self.Sigma_ = None
134        self.GCV_ = None
135        self.return_std = return_std
136
137    def fit(self, X, y, **kwargs):
138        """Fit BayesianRVFLRegressor to training data (X, y).
139
140        Parameters:
141
142            X: {array-like}, shape = [n_samples, n_features]
143                Training vectors, where n_samples is the number
144                of samples and n_features is the number of features.
145
146            y: array-like, shape = [n_samples]
147                Target values.
148
149            **kwargs: additional parameters to be passed to
150                    self.cook_training_set
151
152        Returns:
153
154            self: object
155
156        """
157
158        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
159
160        fit_obj = lmf.beta_Sigma_hat_rvfl(
161            X=scaled_Z,
162            y=centered_y,
163            s=self.s,
164            sigma=self.sigma,
165            fit_intercept=False,
166            return_cov=self.return_std,
167            backend=self.backend,
168        )
169
170        self.beta_ = fit_obj["beta_hat"]
171
172        if self.return_std == True:
173            self.Sigma_ = fit_obj["Sigma_hat"]
174
175        self.GCV_ = fit_obj["GCV"]
176
177        return self
178
179    def predict(self, X, return_std=False, **kwargs):
180        """Predict test data X.
181
182        Parameters:
183
184            X: {array-like}, shape = [n_samples, n_features]
185                Training vectors, where n_samples is the number
186                of samples and n_features is the number of features.
187
188            return_std: {boolean}, standard dev. is returned or not
189
190            **kwargs: additional parameters to be passed to
191                    self.cook_test_set
192
193        Returns:
194
195            model predictions: {array-like}
196
197        """
198
199        if len(X.shape) == 1:  # one observation in the test set only
200            n_features = X.shape[0]
201            new_X = mo.rbind(
202                x=X.reshape(1, n_features),
203                y=np.ones(n_features).reshape(1, n_features),
204                backend=self.backend,
205            )
206
207        self.return_std = return_std
208
209        if self.return_std == False:
210            if len(X.shape) == 1:
211                return (
212                    self.y_mean_
213                    + mo.safe_sparse_dot(
214                        a=self.cook_test_set(new_X, **kwargs),
215                        b=self.beta_,
216                        backend=self.backend,
217                    )
218                )[0]
219
220            return self.y_mean_ + mo.safe_sparse_dot(
221                a=self.cook_test_set(X, **kwargs),
222                b=self.beta_,
223                backend=self.backend,
224            )
225
226        else:  # confidence interval required for preds?
227            if len(X.shape) == 1:
228                Z = self.cook_test_set(new_X, **kwargs)
229
230                pred_obj = lmf.beta_Sigma_hat_rvfl(
231                    s=self.s,
232                    sigma=self.sigma,
233                    X_star=Z,
234                    return_cov=True,
235                    beta_hat_=self.beta_,
236                    Sigma_hat_=self.Sigma_,
237                    backend=self.backend,
238                )
239
240                return (
241                    self.y_mean_ + pred_obj["preds"][0],
242                    pred_obj["preds_std"][0],
243                )
244
245            Z = self.cook_test_set(X, **kwargs)
246
247            pred_obj = lmf.beta_Sigma_hat_rvfl(
248                s=self.s,
249                sigma=self.sigma,
250                X_star=Z,
251                return_cov=True,
252                beta_hat_=self.beta_,
253                Sigma_hat_=self.Sigma_,
254                backend=self.backend,
255            )
256
257            return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"])
258
259    def score(self, X, y, scoring=None, **kwargs):
260        """ Score the model on test set features X and response y. 
261
262        Args:
263        
264            X: {array-like}, shape = [n_samples, n_features]
265                Training vectors, where n_samples is the number 
266                of samples and n_features is the number of features
267
268            y: array-like, shape = [n_samples]
269                Target values
270
271            scoring: str
272                must be in ('explained_variance', 'neg_mean_absolute_error', \
273                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
274                            'neg_median_absolute_error', 'r2')
275            
276            **kwargs: additional parameters to be passed to scoring functions
277               
278        Returns: 
279        
280            model scores: {array-like}
281            
282        """
283
284        preds = self.predict(X)
285
286        if type(preds) == tuple:  # if there are std. devs in the predictions
287            preds = preds[0]
288
289        if scoring is None:
290            scoring = "neg_mean_squared_error"
291
292        # check inputs
293        assert scoring in (
294            "explained_variance",
295            "neg_mean_absolute_error",
296            "neg_mean_squared_error",
297            "neg_mean_squared_log_error",
298            "neg_median_absolute_error",
299            "r2",
300        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
301                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
302                           'neg_median_absolute_error', 'r2')"
303
304        scoring_options = {
305            "explained_variance": skm.explained_variance_score,
306            "neg_mean_absolute_error": skm.median_absolute_error,
307            "neg_mean_squared_error": skm.mean_squared_error,
308            "neg_mean_squared_log_error": skm.mean_squared_log_error,
309            "neg_median_absolute_error": skm.median_absolute_error,
310            "r2": skm.r2_score,
311        }
312
313        return scoring_options[scoring](y, preds, **kwargs)

Bayesian Random Vector Functional Link Network regression with one prior

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original features are included (True) in model''s fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

seed: int
    reproducibility seed for nodes_sim=='uniform'

s: float
    std. dev. of regression parameters in Bayesian Ridge Regression

sigma: float
    std. dev. of residuals in Bayesian Ridge Regression

return_std: boolean
    if True, uncertainty around predictions is evaluated

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

beta_: array-like
    regression''s coefficients

Sigma_: array-like
    covariance of the distribution of fitted parameters

GCV_: float
    Generalized cross-validation error

y_mean_: float
    average response

Examples:

TBD
BayesianRVFLRegressor( n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), seed=123, s=0.1, sigma=0.05, return_std=True, backend='cpu')
 96    def __init__(
 97        self,
 98        n_hidden_features=5,
 99        activation_name="relu",
100        a=0.01,
101        nodes_sim="sobol",
102        bias=True,
103        dropout=0,
104        direct_link=True,
105        n_clusters=2,
106        cluster_encode=True,
107        type_clust="kmeans",
108        type_scaling=("std", "std", "std"),
109        seed=123,
110        s=0.1,
111        sigma=0.05,
112        return_std=True,
113        backend="cpu",
114    ):
115        super().__init__(
116            n_hidden_features=n_hidden_features,
117            activation_name=activation_name,
118            a=a,
119            nodes_sim=nodes_sim,
120            bias=bias,
121            dropout=dropout,
122            direct_link=direct_link,
123            n_clusters=n_clusters,
124            cluster_encode=cluster_encode,
125            type_clust=type_clust,
126            type_scaling=type_scaling,
127            seed=seed,
128            backend=backend,
129        )
130        self.s = s
131        self.sigma = sigma
132        self.beta_ = None
133        self.Sigma_ = None
134        self.GCV_ = None
135        self.return_std = return_std
s
sigma
beta_
Sigma_
GCV_
return_std
def fit(self, X, y, **kwargs):
137    def fit(self, X, y, **kwargs):
138        """Fit BayesianRVFLRegressor to training data (X, y).
139
140        Parameters:
141
142            X: {array-like}, shape = [n_samples, n_features]
143                Training vectors, where n_samples is the number
144                of samples and n_features is the number of features.
145
146            y: array-like, shape = [n_samples]
147                Target values.
148
149            **kwargs: additional parameters to be passed to
150                    self.cook_training_set
151
152        Returns:
153
154            self: object
155
156        """
157
158        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
159
160        fit_obj = lmf.beta_Sigma_hat_rvfl(
161            X=scaled_Z,
162            y=centered_y,
163            s=self.s,
164            sigma=self.sigma,
165            fit_intercept=False,
166            return_cov=self.return_std,
167            backend=self.backend,
168        )
169
170        self.beta_ = fit_obj["beta_hat"]
171
172        if self.return_std == True:
173            self.Sigma_ = fit_obj["Sigma_hat"]
174
175        self.GCV_ = fit_obj["GCV"]
176
177        return self

Fit BayesianRVFLRegressor to training data (X, y).

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set

Returns:

self: object
def predict(self, X, return_std=False, **kwargs):
179    def predict(self, X, return_std=False, **kwargs):
180        """Predict test data X.
181
182        Parameters:
183
184            X: {array-like}, shape = [n_samples, n_features]
185                Training vectors, where n_samples is the number
186                of samples and n_features is the number of features.
187
188            return_std: {boolean}, standard dev. is returned or not
189
190            **kwargs: additional parameters to be passed to
191                    self.cook_test_set
192
193        Returns:
194
195            model predictions: {array-like}
196
197        """
198
199        if len(X.shape) == 1:  # one observation in the test set only
200            n_features = X.shape[0]
201            new_X = mo.rbind(
202                x=X.reshape(1, n_features),
203                y=np.ones(n_features).reshape(1, n_features),
204                backend=self.backend,
205            )
206
207        self.return_std = return_std
208
209        if self.return_std == False:
210            if len(X.shape) == 1:
211                return (
212                    self.y_mean_
213                    + mo.safe_sparse_dot(
214                        a=self.cook_test_set(new_X, **kwargs),
215                        b=self.beta_,
216                        backend=self.backend,
217                    )
218                )[0]
219
220            return self.y_mean_ + mo.safe_sparse_dot(
221                a=self.cook_test_set(X, **kwargs),
222                b=self.beta_,
223                backend=self.backend,
224            )
225
226        else:  # confidence interval required for preds?
227            if len(X.shape) == 1:
228                Z = self.cook_test_set(new_X, **kwargs)
229
230                pred_obj = lmf.beta_Sigma_hat_rvfl(
231                    s=self.s,
232                    sigma=self.sigma,
233                    X_star=Z,
234                    return_cov=True,
235                    beta_hat_=self.beta_,
236                    Sigma_hat_=self.Sigma_,
237                    backend=self.backend,
238                )
239
240                return (
241                    self.y_mean_ + pred_obj["preds"][0],
242                    pred_obj["preds_std"][0],
243                )
244
245            Z = self.cook_test_set(X, **kwargs)
246
247            pred_obj = lmf.beta_Sigma_hat_rvfl(
248                s=self.s,
249                sigma=self.sigma,
250                X_star=Z,
251                return_cov=True,
252                beta_hat_=self.beta_,
253                Sigma_hat_=self.Sigma_,
254                backend=self.backend,
255            )
256
257            return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"])

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

return_std: {boolean}, standard dev. is returned or not

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def score(self, X, y, scoring=None, **kwargs):
259    def score(self, X, y, scoring=None, **kwargs):
260        """ Score the model on test set features X and response y. 
261
262        Args:
263        
264            X: {array-like}, shape = [n_samples, n_features]
265                Training vectors, where n_samples is the number 
266                of samples and n_features is the number of features
267
268            y: array-like, shape = [n_samples]
269                Target values
270
271            scoring: str
272                must be in ('explained_variance', 'neg_mean_absolute_error', \
273                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
274                            'neg_median_absolute_error', 'r2')
275            
276            **kwargs: additional parameters to be passed to scoring functions
277               
278        Returns: 
279        
280            model scores: {array-like}
281            
282        """
283
284        preds = self.predict(X)
285
286        if type(preds) == tuple:  # if there are std. devs in the predictions
287            preds = preds[0]
288
289        if scoring is None:
290            scoring = "neg_mean_squared_error"
291
292        # check inputs
293        assert scoring in (
294            "explained_variance",
295            "neg_mean_absolute_error",
296            "neg_mean_squared_error",
297            "neg_mean_squared_log_error",
298            "neg_median_absolute_error",
299            "r2",
300        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
301                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
302                           'neg_median_absolute_error', 'r2')"
303
304        scoring_options = {
305            "explained_variance": skm.explained_variance_score,
306            "neg_mean_absolute_error": skm.median_absolute_error,
307            "neg_mean_squared_error": skm.mean_squared_error,
308            "neg_mean_squared_log_error": skm.mean_squared_log_error,
309            "neg_median_absolute_error": skm.median_absolute_error,
310            "r2": skm.r2_score,
311        }
312
313        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                             'neg_mean_squared_error', 'neg_mean_squared_log_error',                             'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_predict_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class BayesianRVFL2Regressor(nnetsauce.Base, sklearn.base.RegressorMixin):
 15class BayesianRVFL2Regressor(Base, RegressorMixin):
 16    """Bayesian Random Vector Functional Link Network regression with two priors
 17
 18    Parameters:
 19
 20        n_hidden_features: int
 21            number of nodes in the hidden layer
 22
 23        activation_name: str
 24            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 25
 26        a: float
 27            hyperparameter for 'prelu' or 'elu' activation function
 28
 29        nodes_sim: str
 30            type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform'
 31
 32        bias: boolean
 33            indicates if the hidden layer contains a bias term (True) or not (False)
 34
 35        dropout: float
 36            regularization parameter; (random) percentage of nodes dropped out
 37            of the training
 38
 39        direct_link: boolean
 40            indicates if the original features are included (True) in model''s fitting or not (False)
 41
 42        n_clusters: int
 43            number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering)
 44
 45        cluster_encode: bool
 46            defines how the variable containing clusters is treated (default is one-hot)
 47            if `False`, then labels are used, without one-hot encoding
 48
 49        type_clust: str
 50            type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm')
 51
 52        type_scaling: a tuple of 3 strings
 53            scaling methods for inputs, hidden layer, and clustering respectively
 54            (and when relevant).
 55            Currently available: standardization ('std') or MinMax scaling ('minmax')
 56
 57        seed: int
 58            reproducibility seed for nodes_sim=='uniform'
 59
 60        s1: float
 61            std. dev. of init. regression parameters in Bayesian Ridge Regression
 62
 63        s2: float
 64            std. dev. of augmented regression parameters in Bayesian Ridge Regression
 65
 66        sigma: float
 67            std. dev. of residuals in Bayesian Ridge Regression
 68
 69        return_std: boolean
 70            if True, uncertainty around predictions is evaluated
 71
 72        backend: str
 73            "cpu" or "gpu" or "tpu"
 74
 75    Attributes:
 76
 77        beta_: array-like
 78            regression''s coefficients
 79
 80        Sigma_: array-like
 81            covariance of the distribution of fitted parameters
 82
 83        GCV_: float
 84            Generalized cross-validation error
 85
 86        y_mean_: float
 87            average response
 88
 89    Examples:
 90
 91    ```python
 92    TBD
 93    ```
 94
 95    """
 96
 97    # construct the object -----
 98
 99    def __init__(
100        self,
101        n_hidden_features=5,
102        activation_name="relu",
103        a=0.01,
104        nodes_sim="sobol",
105        bias=True,
106        dropout=0,
107        direct_link=True,
108        n_clusters=0,
109        cluster_encode=True,
110        type_clust="kmeans",
111        type_scaling=("std", "std", "std"),
112        seed=123,
113        s1=0.1,
114        s2=0.1,
115        sigma=0.05,
116        return_std=True,
117        backend="cpu",
118    ):
119        super().__init__(
120            n_hidden_features=n_hidden_features,
121            activation_name=activation_name,
122            a=a,
123            nodes_sim=nodes_sim,
124            bias=bias,
125            dropout=dropout,
126            direct_link=direct_link,
127            n_clusters=n_clusters,
128            cluster_encode=cluster_encode,
129            type_clust=type_clust,
130            type_scaling=type_scaling,
131            seed=seed,
132            backend=backend,
133        )
134
135        self.s1 = s1
136        self.s2 = s2
137        self.sigma = sigma
138        self.beta_ = None
139        self.Sigma_ = None
140        self.GCV_ = None
141        self.return_std = return_std
142
143    def fit(self, X, y, **kwargs):
144        """Fit BayesianRVFL2Regressor to training data (X, y)
145
146        Parameters:
147
148            X: {array-like}, shape = [n_samples, n_features]
149                Training vectors, where n_samples is the number
150                of samples and n_features is the number of features
151
152            y: array-like, shape = [n_samples]
153                Target values
154
155            **kwargs: additional parameters to be passed to
156                    self.cook_training_set
157
158        Returns:
159
160            self: object
161
162        """
163
164        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
165
166        n, p = X.shape
167        q = self.n_hidden_features
168
169        if self.direct_link == True:
170            r = p + self.n_clusters
171
172            block11 = (self.s1**2) * np.eye(r)
173            block12 = np.zeros((r, q))
174            block21 = np.zeros((q, r))
175            block22 = (self.s2**2) * np.eye(q)
176
177            Sigma_prior = mo.rbind(
178                x=mo.cbind(x=block11, y=block12, backend=self.backend),
179                y=mo.cbind(x=block21, y=block22, backend=self.backend),
180                backend=self.backend,
181            )
182
183        else:
184            Sigma_prior = (self.s2**2) * np.eye(q)
185
186        fit_obj = lmf.beta_Sigma_hat_rvfl2(
187            X=scaled_Z,
188            y=centered_y,
189            Sigma=Sigma_prior,
190            sigma=self.sigma,
191            fit_intercept=False,
192            return_cov=self.return_std,
193            backend=self.backend,
194        )
195
196        self.beta_ = fit_obj["beta_hat"]
197
198        if self.return_std == True:
199            self.Sigma_ = fit_obj["Sigma_hat"]
200
201        self.GCV_ = fit_obj["GCV"]
202
203        return self
204
205    def predict(self, X, return_std=False, **kwargs):
206        """Predict test data X.
207
208        Parameters:
209
210            X: {array-like}, shape = [n_samples, n_features]
211                Training vectors, where n_samples is the number
212                of samples and n_features is the number of features.
213
214            return_std: {boolean}, standard dev. is returned or not
215
216            **kwargs: additional parameters to be passed to
217                    self.cook_test_set
218
219        Returns:
220
221            model predictions: {array-like}
222
223        """
224
225        if len(X.shape) == 1:  # one observation in the test set only
226            n_features = X.shape[0]
227            new_X = mo.rbind(
228                x=X.reshape(1, n_features),
229                y=np.ones(n_features).reshape(1, n_features),
230                backend=self.backend,
231            )
232
233        self.return_std = return_std
234
235        if self.return_std == False:
236            if len(X.shape) == 1:
237                return (
238                    self.y_mean_
239                    + mo.safe_sparse_dot(
240                        self.cook_test_set(new_X, **kwargs),
241                        self.beta_,
242                        backend=self.backend,
243                    )
244                )[0]
245
246            return self.y_mean_ + mo.safe_sparse_dot(
247                self.cook_test_set(X, **kwargs),
248                self.beta_,
249                backend=self.backend,
250            )
251
252        else:  # confidence interval required for preds?
253            if len(X.shape) == 1:
254                Z = self.cook_test_set(new_X, **kwargs)
255
256                pred_obj = lmf.beta_Sigma_hat_rvfl2(
257                    X_star=Z,
258                    return_cov=self.return_std,
259                    beta_hat_=self.beta_,
260                    Sigma_hat_=self.Sigma_,
261                    backend=self.backend,
262                )
263
264                return (
265                    self.y_mean_ + pred_obj["preds"][0],
266                    pred_obj["preds_std"][0],
267                )
268
269            Z = self.cook_test_set(X, **kwargs)
270
271            pred_obj = lmf.beta_Sigma_hat_rvfl2(
272                X_star=Z,
273                return_cov=self.return_std,
274                beta_hat_=self.beta_,
275                Sigma_hat_=self.Sigma_,
276                backend=self.backend,
277            )
278
279            return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"])
280
281    def score(self, X, y, scoring=None, **kwargs):
282        """ Score the model on test set features X and response y. 
283
284        Args:
285        
286            X: {array-like}, shape = [n_samples, n_features]
287                Training vectors, where n_samples is the number 
288                of samples and n_features is the number of features
289
290            y: array-like, shape = [n_samples]
291                Target values
292
293            scoring: str
294                must be in ('explained_variance', 'neg_mean_absolute_error', \
295                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
296                            'neg_median_absolute_error', 'r2')
297            
298            **kwargs: additional parameters to be passed to scoring functions
299               
300        Returns: 
301        
302            model scores: {array-like}
303            
304        """
305
306        preds = self.predict(X)
307
308        if type(preds) == tuple:  # if there are std. devs in the predictions
309            preds = preds[0]
310
311        if scoring is None:
312            scoring = "neg_mean_squared_error"
313
314        # check inputs
315        assert scoring in (
316            "explained_variance",
317            "neg_mean_absolute_error",
318            "neg_mean_squared_error",
319            "neg_mean_squared_log_error",
320            "neg_median_absolute_error",
321            "r2",
322        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
323                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
324                           'neg_median_absolute_error', 'r2')"
325
326        scoring_options = {
327            "explained_variance": skm.explained_variance_score,
328            "neg_mean_absolute_error": skm.median_absolute_error,
329            "neg_mean_squared_error": skm.mean_squared_error,
330            "neg_mean_squared_log_error": skm.mean_squared_log_error,
331            "neg_median_absolute_error": skm.median_absolute_error,
332            "r2": skm.r2_score,
333        }
334
335        return scoring_options[scoring](y, preds, **kwargs)

Bayesian Random Vector Functional Link Network regression with two priors

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original features are included (True) in model''s fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

seed: int
    reproducibility seed for nodes_sim=='uniform'

s1: float
    std. dev. of init. regression parameters in Bayesian Ridge Regression

s2: float
    std. dev. of augmented regression parameters in Bayesian Ridge Regression

sigma: float
    std. dev. of residuals in Bayesian Ridge Regression

return_std: boolean
    if True, uncertainty around predictions is evaluated

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

beta_: array-like
    regression''s coefficients

Sigma_: array-like
    covariance of the distribution of fitted parameters

GCV_: float
    Generalized cross-validation error

y_mean_: float
    average response

Examples:

TBD
BayesianRVFL2Regressor( n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=0, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), seed=123, s1=0.1, s2=0.1, sigma=0.05, return_std=True, backend='cpu')
 99    def __init__(
100        self,
101        n_hidden_features=5,
102        activation_name="relu",
103        a=0.01,
104        nodes_sim="sobol",
105        bias=True,
106        dropout=0,
107        direct_link=True,
108        n_clusters=0,
109        cluster_encode=True,
110        type_clust="kmeans",
111        type_scaling=("std", "std", "std"),
112        seed=123,
113        s1=0.1,
114        s2=0.1,
115        sigma=0.05,
116        return_std=True,
117        backend="cpu",
118    ):
119        super().__init__(
120            n_hidden_features=n_hidden_features,
121            activation_name=activation_name,
122            a=a,
123            nodes_sim=nodes_sim,
124            bias=bias,
125            dropout=dropout,
126            direct_link=direct_link,
127            n_clusters=n_clusters,
128            cluster_encode=cluster_encode,
129            type_clust=type_clust,
130            type_scaling=type_scaling,
131            seed=seed,
132            backend=backend,
133        )
134
135        self.s1 = s1
136        self.s2 = s2
137        self.sigma = sigma
138        self.beta_ = None
139        self.Sigma_ = None
140        self.GCV_ = None
141        self.return_std = return_std
s1
s2
sigma
beta_
Sigma_
GCV_
return_std
def fit(self, X, y, **kwargs):
143    def fit(self, X, y, **kwargs):
144        """Fit BayesianRVFL2Regressor to training data (X, y)
145
146        Parameters:
147
148            X: {array-like}, shape = [n_samples, n_features]
149                Training vectors, where n_samples is the number
150                of samples and n_features is the number of features
151
152            y: array-like, shape = [n_samples]
153                Target values
154
155            **kwargs: additional parameters to be passed to
156                    self.cook_training_set
157
158        Returns:
159
160            self: object
161
162        """
163
164        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
165
166        n, p = X.shape
167        q = self.n_hidden_features
168
169        if self.direct_link == True:
170            r = p + self.n_clusters
171
172            block11 = (self.s1**2) * np.eye(r)
173            block12 = np.zeros((r, q))
174            block21 = np.zeros((q, r))
175            block22 = (self.s2**2) * np.eye(q)
176
177            Sigma_prior = mo.rbind(
178                x=mo.cbind(x=block11, y=block12, backend=self.backend),
179                y=mo.cbind(x=block21, y=block22, backend=self.backend),
180                backend=self.backend,
181            )
182
183        else:
184            Sigma_prior = (self.s2**2) * np.eye(q)
185
186        fit_obj = lmf.beta_Sigma_hat_rvfl2(
187            X=scaled_Z,
188            y=centered_y,
189            Sigma=Sigma_prior,
190            sigma=self.sigma,
191            fit_intercept=False,
192            return_cov=self.return_std,
193            backend=self.backend,
194        )
195
196        self.beta_ = fit_obj["beta_hat"]
197
198        if self.return_std == True:
199            self.Sigma_ = fit_obj["Sigma_hat"]
200
201        self.GCV_ = fit_obj["GCV"]
202
203        return self

Fit BayesianRVFL2Regressor to training data (X, y)

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

**kwargs: additional parameters to be passed to
        self.cook_training_set

Returns:

self: object
def predict(self, X, return_std=False, **kwargs):
205    def predict(self, X, return_std=False, **kwargs):
206        """Predict test data X.
207
208        Parameters:
209
210            X: {array-like}, shape = [n_samples, n_features]
211                Training vectors, where n_samples is the number
212                of samples and n_features is the number of features.
213
214            return_std: {boolean}, standard dev. is returned or not
215
216            **kwargs: additional parameters to be passed to
217                    self.cook_test_set
218
219        Returns:
220
221            model predictions: {array-like}
222
223        """
224
225        if len(X.shape) == 1:  # one observation in the test set only
226            n_features = X.shape[0]
227            new_X = mo.rbind(
228                x=X.reshape(1, n_features),
229                y=np.ones(n_features).reshape(1, n_features),
230                backend=self.backend,
231            )
232
233        self.return_std = return_std
234
235        if self.return_std == False:
236            if len(X.shape) == 1:
237                return (
238                    self.y_mean_
239                    + mo.safe_sparse_dot(
240                        self.cook_test_set(new_X, **kwargs),
241                        self.beta_,
242                        backend=self.backend,
243                    )
244                )[0]
245
246            return self.y_mean_ + mo.safe_sparse_dot(
247                self.cook_test_set(X, **kwargs),
248                self.beta_,
249                backend=self.backend,
250            )
251
252        else:  # confidence interval required for preds?
253            if len(X.shape) == 1:
254                Z = self.cook_test_set(new_X, **kwargs)
255
256                pred_obj = lmf.beta_Sigma_hat_rvfl2(
257                    X_star=Z,
258                    return_cov=self.return_std,
259                    beta_hat_=self.beta_,
260                    Sigma_hat_=self.Sigma_,
261                    backend=self.backend,
262                )
263
264                return (
265                    self.y_mean_ + pred_obj["preds"][0],
266                    pred_obj["preds_std"][0],
267                )
268
269            Z = self.cook_test_set(X, **kwargs)
270
271            pred_obj = lmf.beta_Sigma_hat_rvfl2(
272                X_star=Z,
273                return_cov=self.return_std,
274                beta_hat_=self.beta_,
275                Sigma_hat_=self.Sigma_,
276                backend=self.backend,
277            )
278
279            return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"])

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

return_std: {boolean}, standard dev. is returned or not

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def score(self, X, y, scoring=None, **kwargs):
281    def score(self, X, y, scoring=None, **kwargs):
282        """ Score the model on test set features X and response y. 
283
284        Args:
285        
286            X: {array-like}, shape = [n_samples, n_features]
287                Training vectors, where n_samples is the number 
288                of samples and n_features is the number of features
289
290            y: array-like, shape = [n_samples]
291                Target values
292
293            scoring: str
294                must be in ('explained_variance', 'neg_mean_absolute_error', \
295                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
296                            'neg_median_absolute_error', 'r2')
297            
298            **kwargs: additional parameters to be passed to scoring functions
299               
300        Returns: 
301        
302            model scores: {array-like}
303            
304        """
305
306        preds = self.predict(X)
307
308        if type(preds) == tuple:  # if there are std. devs in the predictions
309            preds = preds[0]
310
311        if scoring is None:
312            scoring = "neg_mean_squared_error"
313
314        # check inputs
315        assert scoring in (
316            "explained_variance",
317            "neg_mean_absolute_error",
318            "neg_mean_squared_error",
319            "neg_mean_squared_log_error",
320            "neg_median_absolute_error",
321            "r2",
322        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
323                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
324                           'neg_median_absolute_error', 'r2')"
325
326        scoring_options = {
327            "explained_variance": skm.explained_variance_score,
328            "neg_mean_absolute_error": skm.median_absolute_error,
329            "neg_mean_squared_error": skm.mean_squared_error,
330            "neg_mean_squared_log_error": skm.mean_squared_log_error,
331            "neg_median_absolute_error": skm.median_absolute_error,
332            "r2": skm.r2_score,
333        }
334
335        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                             'neg_mean_squared_error', 'neg_mean_squared_log_error',                             'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_predict_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class CustomClassifier(nnetsauce.custom.custom.Custom, sklearn.base.ClassifierMixin):
 13class CustomClassifier(Custom, ClassifierMixin):
 14    """Custom Classification model
 15
 16    Attributes:
 17
 18        obj: object
 19            any object containing a method fit (obj.fit()) and a method predict
 20            (obj.predict())
 21
 22        n_hidden_features: int
 23            number of nodes in the hidden layer
 24
 25        activation_name: str
 26            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 27
 28        a: float
 29            hyperparameter for 'prelu' or 'elu' activation function
 30
 31        nodes_sim: str
 32            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 33            'uniform'
 34
 35        bias: boolean
 36            indicates if the hidden layer contains a bias term (True) or not
 37            (False)
 38
 39        dropout: float
 40            regularization parameter; (random) percentage of nodes dropped out
 41            of the training
 42
 43        direct_link: boolean
 44            indicates if the original predictors are included (True) in model''s
 45            fitting or not (False)
 46
 47        n_clusters: int
 48            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 49                no clustering)
 50
 51        cluster_encode: bool
 52            defines how the variable containing clusters is treated (default is one-hot)
 53            if `False`, then labels are used, without one-hot encoding
 54
 55        type_clust: str
 56            type of clustering method: currently k-means ('kmeans') or Gaussian
 57            Mixture Model ('gmm')
 58
 59        type_scaling: a tuple of 3 strings
 60            scaling methods for inputs, hidden layer, and clustering respectively
 61            (and when relevant).
 62            Currently available: standardization ('std') or MinMax scaling ('minmax')
 63
 64        col_sample: float
 65            percentage of covariates randomly chosen for training
 66
 67        row_sample: float
 68            percentage of rows chosen for training, by stratified bootstrapping
 69
 70        seed: int
 71            reproducibility seed for nodes_sim=='uniform'
 72
 73        backend: str
 74            "cpu" or "gpu" or "tpu"
 75
 76    Examples:
 77
 78    ```python
 79    import nnetsauce as ns
 80    from sklearn.ensemble import RandomForestClassifier
 81    from sklearn.model_selection import train_test_split
 82    from sklearn.datasets import load_digits
 83    from time import time
 84
 85    digits = load_digits()
 86    X = digits.data
 87    y = digits.target
 88    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
 89                                                        random_state=123)
 90
 91    # layer 1 (base layer) ----
 92    layer1_regr = RandomForestClassifier(n_estimators=10, random_state=123)
 93
 94    start = time()
 95
 96    layer1_regr.fit(X_train, y_train)
 97
 98    # Accuracy in layer 1
 99    print(layer1_regr.score(X_test, y_test))
100
101    # layer 2 using layer 1 ----
102    layer2_regr = ns.CustomClassifier(obj = layer1_regr, n_hidden_features=5,
103                            direct_link=True, bias=True,
104                            nodes_sim='uniform', activation_name='relu',
105                            n_clusters=2, seed=123)
106    layer2_regr.fit(X_train, y_train)
107
108    # Accuracy in layer 2
109    print(layer2_regr.score(X_test, y_test))
110
111    # layer 3 using layer 2 ----
112    layer3_regr = ns.CustomClassifier(obj = layer2_regr, n_hidden_features=10,
113                            direct_link=True, bias=True, dropout=0.7,
114                            nodes_sim='uniform', activation_name='relu',
115                            n_clusters=2, seed=123)
116    layer3_regr.fit(X_train, y_train)
117
118    # Accuracy in layer 3
119    print(layer3_regr.score(X_test, y_test))
120
121    print(f"Elapsed {time() - start}")
122    ```
123
124    """
125
126    # construct the object -----
127
128    def __init__(
129        self,
130        obj,
131        n_hidden_features=5,
132        activation_name="relu",
133        a=0.01,
134        nodes_sim="sobol",
135        bias=True,
136        dropout=0,
137        direct_link=True,
138        n_clusters=2,
139        cluster_encode=True,
140        type_clust="kmeans",
141        type_scaling=("std", "std", "std"),
142        col_sample=1,
143        row_sample=1,
144        seed=123,
145        backend="cpu",
146    ):
147        super().__init__(
148            obj=obj,
149            n_hidden_features=n_hidden_features,
150            activation_name=activation_name,
151            a=a,
152            nodes_sim=nodes_sim,
153            bias=bias,
154            dropout=dropout,
155            direct_link=direct_link,
156            n_clusters=n_clusters,
157            cluster_encode=cluster_encode,
158            type_clust=type_clust,
159            type_scaling=type_scaling,
160            col_sample=col_sample,
161            row_sample=row_sample,
162            seed=seed,
163            backend=backend,
164        )
165
166        self.type_fit = "classification"
167
168    def fit(self, X, y, sample_weight=None, **kwargs):
169        """Fit custom model to training data (X, y).
170
171        Parameters:
172
173            X: {array-like}, shape = [n_samples, n_features]
174                Training vectors, where n_samples is the number
175                of samples and n_features is the number of features.
176
177            y: array-like, shape = [n_samples]
178                Target values.
179
180            **kwargs: additional parameters to be passed to
181                        self.cook_training_set or self.obj.fit
182
183        Returns:
184
185            self: object
186        """
187
188        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
189
190        # if sample_weights, else: (must use self.row_index)
191        if sample_weight is not None:
192            self.obj.fit(
193                scaled_Z,
194                output_y,
195                sample_weight=np.ravel(sample_weight, order="C")[
196                    self.index_row_
197                ],
198                # **kwargs
199            )
200
201            return self
202
203        # if sample_weight is None:
204        self.obj.fit(scaled_Z, output_y)
205
206        return self
207
208    def predict(self, X, **kwargs):
209        """Predict test data X.
210
211        Parameters:
212
213            X: {array-like}, shape = [n_samples, n_features]
214                Training vectors, where n_samples is the number
215                of samples and n_features is the number of features.
216
217            **kwargs: additional parameters to be passed to
218                    self.cook_test_set
219
220        Returns:
221
222            model predictions: {array-like}
223        """
224
225        if len(X.shape) == 1:
226            n_features = X.shape[0]
227            new_X = mo.rbind(
228                X.reshape(1, n_features),
229                np.ones(n_features).reshape(1, n_features),
230            )
231
232            return (
233                self.obj.predict(self.cook_test_set(new_X, **kwargs), **kwargs)
234            )[0]
235
236        return self.obj.predict(self.cook_test_set(X, **kwargs), **kwargs)
237
238    def predict_proba(self, X, **kwargs):
239        """Predict probabilities for test data X.
240
241        Args:
242
243            X: {array-like}, shape = [n_samples, n_features]
244                Training vectors, where n_samples is the number
245                of samples and n_features is the number of features.
246
247            **kwargs: additional parameters to be passed to
248                    self.cook_test_set
249
250        Returns:
251
252            probability estimates for test data: {array-like}
253        """
254
255        if len(X.shape) == 1:
256            n_features = X.shape[0]
257            new_X = mo.rbind(
258                X.reshape(1, n_features),
259                np.ones(n_features).reshape(1, n_features),
260            )
261
262            return (
263                self.obj.predict_proba(
264                    self.cook_test_set(new_X, **kwargs), **kwargs
265                )
266            )[0]
267
268        return self.obj.predict_proba(self.cook_test_set(X, **kwargs), **kwargs)
269
270    def score(self, X, y, scoring=None, **kwargs):
271        """ Score the model on test set features X and response y. 
272
273        Args:
274    
275            X: {array-like}, shape = [n_samples, n_features]
276                Training vectors, where n_samples is the number 
277                of samples and n_features is the number of features
278
279            y: array-like, shape = [n_samples]
280                Target values
281
282            scoring: str
283                must be in ('explained_variance', 'neg_mean_absolute_error', \
284                    'neg_mean_squared_error', 'neg_mean_squared_log_error', \
285                    'neg_median_absolute_error', 'r2')
286    
287            **kwargs: 
288                additional parameters to be passed to scoring functions
289            
290        Returns: 
291    
292            model scores: {array-like}
293
294        """
295
296        preds = self.predict(X)
297
298        if scoring is None:
299            scoring = "accuracy"
300
301        # check inputs
302        assert scoring in (
303            "accuracy",
304            "average_precision",
305            "brier_score_loss",
306            "f1",
307            "f1_micro",
308            "f1_macro",
309            "f1_weighted",
310            "f1_samples",
311            "neg_log_loss",
312            "precision",
313            "recall",
314            "roc_auc",
315        ), "'scoring' should be in ('accuracy', 'average_precision', \
316                           'brier_score_loss', 'f1', 'f1_micro', \
317                           'f1_macro', 'f1_weighted',  'f1_samples', \
318                           'neg_log_loss', 'precision', 'recall', \
319                           'roc_auc')"
320
321        scoring_options = {
322            "accuracy": skm2.accuracy_score,
323            "average_precision": skm2.average_precision_score,
324            "brier_score_loss": skm2.brier_score_loss,
325            "f1": skm2.f1_score,
326            "f1_micro": skm2.f1_score,
327            "f1_macro": skm2.f1_score,
328            "f1_weighted": skm2.f1_score,
329            "f1_samples": skm2.f1_score,
330            "neg_log_loss": skm2.log_loss,
331            "precision": skm2.precision_score,
332            "recall": skm2.recall_score,
333            "roc_auc": skm2.roc_auc_score,
334        }
335
336        return scoring_options[scoring](y, preds, **kwargs)

Custom Classification model

Attributes:

obj: object
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict())

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model''s
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Examples:

import nnetsauce as ns
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_digits
from time import time

digits = load_digits()
X = digits.data
y = digits.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
                                                    random_state=123)

# layer 1 (base layer) ----
layer1_regr = RandomForestClassifier(n_estimators=10, random_state=123)

start = time()

layer1_regr.fit(X_train, y_train)

# Accuracy in layer 1
print(layer1_regr.score(X_test, y_test))

# layer 2 using layer 1 ----
layer2_regr = ns.CustomClassifier(obj = layer1_regr, n_hidden_features=5,
                        direct_link=True, bias=True,
                        nodes_sim='uniform', activation_name='relu',
                        n_clusters=2, seed=123)
layer2_regr.fit(X_train, y_train)

# Accuracy in layer 2
print(layer2_regr.score(X_test, y_test))

# layer 3 using layer 2 ----
layer3_regr = ns.CustomClassifier(obj = layer2_regr, n_hidden_features=10,
                        direct_link=True, bias=True, dropout=0.7,
                        nodes_sim='uniform', activation_name='relu',
                        n_clusters=2, seed=123)
layer3_regr.fit(X_train, y_train)

# Accuracy in layer 3
print(layer3_regr.score(X_test, y_test))

print(f"Elapsed {time() - start}")
CustomClassifier( obj, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
128    def __init__(
129        self,
130        obj,
131        n_hidden_features=5,
132        activation_name="relu",
133        a=0.01,
134        nodes_sim="sobol",
135        bias=True,
136        dropout=0,
137        direct_link=True,
138        n_clusters=2,
139        cluster_encode=True,
140        type_clust="kmeans",
141        type_scaling=("std", "std", "std"),
142        col_sample=1,
143        row_sample=1,
144        seed=123,
145        backend="cpu",
146    ):
147        super().__init__(
148            obj=obj,
149            n_hidden_features=n_hidden_features,
150            activation_name=activation_name,
151            a=a,
152            nodes_sim=nodes_sim,
153            bias=bias,
154            dropout=dropout,
155            direct_link=direct_link,
156            n_clusters=n_clusters,
157            cluster_encode=cluster_encode,
158            type_clust=type_clust,
159            type_scaling=type_scaling,
160            col_sample=col_sample,
161            row_sample=row_sample,
162            seed=seed,
163            backend=backend,
164        )
165
166        self.type_fit = "classification"
type_fit
def fit(self, X, y, sample_weight=None, **kwargs):
168    def fit(self, X, y, sample_weight=None, **kwargs):
169        """Fit custom model to training data (X, y).
170
171        Parameters:
172
173            X: {array-like}, shape = [n_samples, n_features]
174                Training vectors, where n_samples is the number
175                of samples and n_features is the number of features.
176
177            y: array-like, shape = [n_samples]
178                Target values.
179
180            **kwargs: additional parameters to be passed to
181                        self.cook_training_set or self.obj.fit
182
183        Returns:
184
185            self: object
186        """
187
188        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
189
190        # if sample_weights, else: (must use self.row_index)
191        if sample_weight is not None:
192            self.obj.fit(
193                scaled_Z,
194                output_y,
195                sample_weight=np.ravel(sample_weight, order="C")[
196                    self.index_row_
197                ],
198                # **kwargs
199            )
200
201            return self
202
203        # if sample_weight is None:
204        self.obj.fit(scaled_Z, output_y)
205
206        return self

Fit custom model to training data (X, y).

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
            self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
208    def predict(self, X, **kwargs):
209        """Predict test data X.
210
211        Parameters:
212
213            X: {array-like}, shape = [n_samples, n_features]
214                Training vectors, where n_samples is the number
215                of samples and n_features is the number of features.
216
217            **kwargs: additional parameters to be passed to
218                    self.cook_test_set
219
220        Returns:
221
222            model predictions: {array-like}
223        """
224
225        if len(X.shape) == 1:
226            n_features = X.shape[0]
227            new_X = mo.rbind(
228                X.reshape(1, n_features),
229                np.ones(n_features).reshape(1, n_features),
230            )
231
232            return (
233                self.obj.predict(self.cook_test_set(new_X, **kwargs), **kwargs)
234            )[0]
235
236        return self.obj.predict(self.cook_test_set(X, **kwargs), **kwargs)

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
238    def predict_proba(self, X, **kwargs):
239        """Predict probabilities for test data X.
240
241        Args:
242
243            X: {array-like}, shape = [n_samples, n_features]
244                Training vectors, where n_samples is the number
245                of samples and n_features is the number of features.
246
247            **kwargs: additional parameters to be passed to
248                    self.cook_test_set
249
250        Returns:
251
252            probability estimates for test data: {array-like}
253        """
254
255        if len(X.shape) == 1:
256            n_features = X.shape[0]
257            new_X = mo.rbind(
258                X.reshape(1, n_features),
259                np.ones(n_features).reshape(1, n_features),
260            )
261
262            return (
263                self.obj.predict_proba(
264                    self.cook_test_set(new_X, **kwargs), **kwargs
265                )
266            )[0]
267
268        return self.obj.predict_proba(self.cook_test_set(X, **kwargs), **kwargs)

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
def score(self, X, y, scoring=None, **kwargs):
270    def score(self, X, y, scoring=None, **kwargs):
271        """ Score the model on test set features X and response y. 
272
273        Args:
274    
275            X: {array-like}, shape = [n_samples, n_features]
276                Training vectors, where n_samples is the number 
277                of samples and n_features is the number of features
278
279            y: array-like, shape = [n_samples]
280                Target values
281
282            scoring: str
283                must be in ('explained_variance', 'neg_mean_absolute_error', \
284                    'neg_mean_squared_error', 'neg_mean_squared_log_error', \
285                    'neg_median_absolute_error', 'r2')
286    
287            **kwargs: 
288                additional parameters to be passed to scoring functions
289            
290        Returns: 
291    
292            model scores: {array-like}
293
294        """
295
296        preds = self.predict(X)
297
298        if scoring is None:
299            scoring = "accuracy"
300
301        # check inputs
302        assert scoring in (
303            "accuracy",
304            "average_precision",
305            "brier_score_loss",
306            "f1",
307            "f1_micro",
308            "f1_macro",
309            "f1_weighted",
310            "f1_samples",
311            "neg_log_loss",
312            "precision",
313            "recall",
314            "roc_auc",
315        ), "'scoring' should be in ('accuracy', 'average_precision', \
316                           'brier_score_loss', 'f1', 'f1_micro', \
317                           'f1_macro', 'f1_weighted',  'f1_samples', \
318                           'neg_log_loss', 'precision', 'recall', \
319                           'roc_auc')"
320
321        scoring_options = {
322            "accuracy": skm2.accuracy_score,
323            "average_precision": skm2.average_precision_score,
324            "brier_score_loss": skm2.brier_score_loss,
325            "f1": skm2.f1_score,
326            "f1_micro": skm2.f1_score,
327            "f1_macro": skm2.f1_score,
328            "f1_weighted": skm2.f1_score,
329            "f1_samples": skm2.f1_score,
330            "neg_log_loss": skm2.log_loss,
331            "precision": skm2.precision_score,
332            "recall": skm2.recall_score,
333            "roc_auc": skm2.roc_auc_score,
334        }
335
336        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                     'neg_mean_squared_error', 'neg_mean_squared_log_error',                     'neg_median_absolute_error', 'r2')

**kwargs: 
    additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class CustomRegressor(nnetsauce.custom.custom.Custom, sklearn.base.RegressorMixin):
 14class CustomRegressor(Custom, RegressorMixin):
 15    """Custom Regression model
 16
 17    This class is used to 'augment' any regression model with transformed features.
 18
 19    Parameters:
 20
 21        obj: object
 22            any object containing a method fit (obj.fit()) and a method predict
 23            (obj.predict())
 24
 25        n_hidden_features: int
 26            number of nodes in the hidden layer
 27
 28        activation_name: str
 29            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 30
 31        a: float
 32            hyperparameter for 'prelu' or 'elu' activation function
 33
 34        nodes_sim: str
 35            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 36            'uniform'
 37
 38        bias: boolean
 39            indicates if the hidden layer contains a bias term (True) or not
 40            (False)
 41
 42        dropout: float
 43            regularization parameter; (random) percentage of nodes dropped out
 44            of the training
 45
 46        direct_link: boolean
 47            indicates if the original predictors are included (True) in model's
 48            fitting or not (False)
 49
 50        n_clusters: int
 51            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 52                no clustering)
 53
 54        cluster_encode: bool
 55            defines how the variable containing clusters is treated (default is one-hot)
 56            if `False`, then labels are used, without one-hot encoding
 57
 58        type_clust: str
 59            type of clustering method: currently k-means ('kmeans') or Gaussian
 60            Mixture Model ('gmm')
 61
 62        type_scaling: a tuple of 3 strings
 63            scaling methods for inputs, hidden layer, and clustering respectively
 64            (and when relevant).
 65            Currently available: standardization ('std') or MinMax scaling ('minmax')
 66
 67        col_sample: float
 68            percentage of covariates randomly chosen for training
 69
 70        row_sample: float
 71            percentage of rows chosen for training, by stratified bootstrapping
 72
 73        seed: int
 74            reproducibility seed for nodes_sim=='uniform'
 75
 76        type_fit: str
 77            'regression'
 78
 79        backend: str
 80            "cpu" or "gpu" or "tpu"
 81
 82    Examples:
 83
 84    ```python
 85    TBD
 86    ```
 87
 88    """
 89
 90    # construct the object -----
 91
 92    def __init__(
 93        self,
 94        obj,
 95        n_hidden_features=5,
 96        activation_name="relu",
 97        a=0.01,
 98        nodes_sim="sobol",
 99        bias=True,
100        dropout=0,
101        direct_link=True,
102        n_clusters=2,
103        cluster_encode=True,
104        type_clust="kmeans",
105        type_scaling=("std", "std", "std"),
106        col_sample=1,
107        row_sample=1,
108        seed=123,
109        backend="cpu",
110    ):
111        super().__init__(
112            obj=obj,
113            n_hidden_features=n_hidden_features,
114            activation_name=activation_name,
115            a=a,
116            nodes_sim=nodes_sim,
117            bias=bias,
118            dropout=dropout,
119            direct_link=direct_link,
120            n_clusters=n_clusters,
121            cluster_encode=cluster_encode,
122            type_clust=type_clust,
123            type_scaling=type_scaling,
124            col_sample=col_sample,
125            row_sample=row_sample,
126            seed=seed,
127            backend=backend,
128        )
129
130        self.type_fit = "regression"
131
132    def fit(self, X, y, sample_weight=None, **kwargs):
133        """Fit custom model to training data (X, y).
134
135        Parameters:
136
137            X: {array-like}, shape = [n_samples, n_features]
138                Training vectors, where n_samples is the number
139                of samples and n_features is the number of features.
140
141            y: array-like, shape = [n_samples]
142                Target values.
143
144            **kwargs: additional parameters to be passed to
145                self.cook_training_set or self.obj.fit
146
147        Returns:
148
149            self: object
150
151        """
152
153        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
154
155        # if sample_weights, else: (must use self.row_index)
156        if sample_weight is not None:
157            self.obj.fit(
158                scaled_Z,
159                centered_y,
160                sample_weight=np.ravel(sample_weight, order="C")[
161                    self.index_row
162                ],
163                **kwargs
164            )
165
166            return self
167
168        self.obj.fit(scaled_Z, centered_y, **kwargs)
169
170        return self
171
172    def predict(self, X, **kwargs):
173        """Predict test data X.
174
175        Parameters:
176
177            X: {array-like}, shape = [n_samples, n_features]
178                Training vectors, where n_samples is the number
179                of samples and n_features is the number of features.
180
181                **kwargs: additional parameters to be passed to
182                    self.cook_test_set
183
184        Returns:
185
186            model predictions: {array-like}
187
188        """
189
190        if len(X.shape) == 1:
191            n_features = X.shape[0]
192            new_X = mo.rbind(
193                X.reshape(1, n_features),
194                np.ones(n_features).reshape(1, n_features),
195            )
196
197            return (
198                self.y_mean_
199                + self.obj.predict(
200                    self.cook_test_set(new_X, **kwargs), **kwargs
201                )
202            )[0]
203
204        return self.y_mean_ + self.obj.predict(
205            self.cook_test_set(X, **kwargs), **kwargs
206        )
207
208    def score(self, X, y, scoring=None, **kwargs):
209        """ Score the model on test set features X and response y. 
210
211        Parameters: 
212    
213            X: {array-like}, shape = [n_samples, n_features]
214                Training vectors, where n_samples is the number 
215                of samples and n_features is the number of features
216
217            y: array-like, shape = [n_samples]
218                Target values
219
220            scoring: str
221                must be in ('explained_variance', 'neg_mean_absolute_error', \
222                    'neg_mean_squared_error', 'neg_mean_squared_log_error', \
223                    'neg_median_absolute_error', 'r2')
224    
225            **kwargs: 
226                additional parameters to be passed to scoring functions
227            
228        Returns:
229    
230            model scores: {array-like}
231            
232        """
233
234        preds = self.predict(X)
235
236        if type(preds) == tuple:  # if there are std. devs in the predictions
237            preds = preds[0]
238
239        if scoring is None:
240            scoring = "neg_root_mean_squared_error"
241
242        # check inputs
243        assert scoring in (
244            "explained_variance",
245            "neg_mean_absolute_error",
246            "neg_mean_squared_error",
247            "neg_mean_squared_log_error",
248            "neg_median_absolute_error",
249            "r2",
250            "neg_root_mean_squared_error",
251        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
252                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
253                           'neg_median_absolute_error', 'r2', 'neg_root_mean_squared_error')"
254
255        scoring_options = {
256            "explained_variance": skm2.explained_variance_score,
257            "neg_mean_absolute_error": skm2.mean_absolute_error,
258            "neg_mean_squared_error": skm2.mean_squared_error,
259            "neg_mean_squared_log_error": skm2.mean_squared_log_error,
260            "neg_median_absolute_error": skm2.median_absolute_error,
261            "r2": skm2.r2_score,
262            "neg_root_mean_squared_error": partial(
263                skm2.mean_squared_error, squared=False
264            ),
265        }
266
267        return scoring_options[scoring](y, preds, **kwargs)

Custom Regression model

This class is used to 'augment' any regression model with transformed features.

Parameters:

obj: object
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict())

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform'

type_fit: str
    'regression'

backend: str
    "cpu" or "gpu" or "tpu"

Examples:

TBD
CustomRegressor( obj, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
 92    def __init__(
 93        self,
 94        obj,
 95        n_hidden_features=5,
 96        activation_name="relu",
 97        a=0.01,
 98        nodes_sim="sobol",
 99        bias=True,
100        dropout=0,
101        direct_link=True,
102        n_clusters=2,
103        cluster_encode=True,
104        type_clust="kmeans",
105        type_scaling=("std", "std", "std"),
106        col_sample=1,
107        row_sample=1,
108        seed=123,
109        backend="cpu",
110    ):
111        super().__init__(
112            obj=obj,
113            n_hidden_features=n_hidden_features,
114            activation_name=activation_name,
115            a=a,
116            nodes_sim=nodes_sim,
117            bias=bias,
118            dropout=dropout,
119            direct_link=direct_link,
120            n_clusters=n_clusters,
121            cluster_encode=cluster_encode,
122            type_clust=type_clust,
123            type_scaling=type_scaling,
124            col_sample=col_sample,
125            row_sample=row_sample,
126            seed=seed,
127            backend=backend,
128        )
129
130        self.type_fit = "regression"
type_fit
def fit(self, X, y, sample_weight=None, **kwargs):
132    def fit(self, X, y, sample_weight=None, **kwargs):
133        """Fit custom model to training data (X, y).
134
135        Parameters:
136
137            X: {array-like}, shape = [n_samples, n_features]
138                Training vectors, where n_samples is the number
139                of samples and n_features is the number of features.
140
141            y: array-like, shape = [n_samples]
142                Target values.
143
144            **kwargs: additional parameters to be passed to
145                self.cook_training_set or self.obj.fit
146
147        Returns:
148
149            self: object
150
151        """
152
153        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
154
155        # if sample_weights, else: (must use self.row_index)
156        if sample_weight is not None:
157            self.obj.fit(
158                scaled_Z,
159                centered_y,
160                sample_weight=np.ravel(sample_weight, order="C")[
161                    self.index_row
162                ],
163                **kwargs
164            )
165
166            return self
167
168        self.obj.fit(scaled_Z, centered_y, **kwargs)
169
170        return self

Fit custom model to training data (X, y).

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
    self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
172    def predict(self, X, **kwargs):
173        """Predict test data X.
174
175        Parameters:
176
177            X: {array-like}, shape = [n_samples, n_features]
178                Training vectors, where n_samples is the number
179                of samples and n_features is the number of features.
180
181                **kwargs: additional parameters to be passed to
182                    self.cook_test_set
183
184        Returns:
185
186            model predictions: {array-like}
187
188        """
189
190        if len(X.shape) == 1:
191            n_features = X.shape[0]
192            new_X = mo.rbind(
193                X.reshape(1, n_features),
194                np.ones(n_features).reshape(1, n_features),
195            )
196
197            return (
198                self.y_mean_
199                + self.obj.predict(
200                    self.cook_test_set(new_X, **kwargs), **kwargs
201                )
202            )[0]
203
204        return self.y_mean_ + self.obj.predict(
205            self.cook_test_set(X, **kwargs), **kwargs
206        )

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

    **kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def score(self, X, y, scoring=None, **kwargs):
208    def score(self, X, y, scoring=None, **kwargs):
209        """ Score the model on test set features X and response y. 
210
211        Parameters: 
212    
213            X: {array-like}, shape = [n_samples, n_features]
214                Training vectors, where n_samples is the number 
215                of samples and n_features is the number of features
216
217            y: array-like, shape = [n_samples]
218                Target values
219
220            scoring: str
221                must be in ('explained_variance', 'neg_mean_absolute_error', \
222                    'neg_mean_squared_error', 'neg_mean_squared_log_error', \
223                    'neg_median_absolute_error', 'r2')
224    
225            **kwargs: 
226                additional parameters to be passed to scoring functions
227            
228        Returns:
229    
230            model scores: {array-like}
231            
232        """
233
234        preds = self.predict(X)
235
236        if type(preds) == tuple:  # if there are std. devs in the predictions
237            preds = preds[0]
238
239        if scoring is None:
240            scoring = "neg_root_mean_squared_error"
241
242        # check inputs
243        assert scoring in (
244            "explained_variance",
245            "neg_mean_absolute_error",
246            "neg_mean_squared_error",
247            "neg_mean_squared_log_error",
248            "neg_median_absolute_error",
249            "r2",
250            "neg_root_mean_squared_error",
251        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
252                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
253                           'neg_median_absolute_error', 'r2', 'neg_root_mean_squared_error')"
254
255        scoring_options = {
256            "explained_variance": skm2.explained_variance_score,
257            "neg_mean_absolute_error": skm2.mean_absolute_error,
258            "neg_mean_squared_error": skm2.mean_squared_error,
259            "neg_mean_squared_log_error": skm2.mean_squared_log_error,
260            "neg_median_absolute_error": skm2.median_absolute_error,
261            "r2": skm2.r2_score,
262            "neg_root_mean_squared_error": partial(
263                skm2.mean_squared_error, squared=False
264            ),
265        }
266
267        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                     'neg_mean_squared_error', 'neg_mean_squared_log_error',                     'neg_median_absolute_error', 'r2')

**kwargs: 
    additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class DeepClassifier(nnetsauce.CustomClassifier, sklearn.base.ClassifierMixin):
 12class DeepClassifier(CustomClassifier, ClassifierMixin):
 13    """
 14    Deep Classifier
 15
 16    Parameters
 17    ----------
 18    verbose : int, optional (default=0)
 19        Monitor progress when fitting.
 20
 21    Examples
 22    --------
 23    >>> import nnetsauce as ns
 24    >>> from sklearn.datasets import load_breast_cancer
 25    >>> from sklearn.model_selection import train_test_split
 26    >>> from sklearn.linear_model import LogisticRegressionCV
 27    >>> data = load_breast_cancer()
 28    >>> X = data.data
 29    >>> y= data.target
 30    >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
 31    >>> obj = LogisticRegressionCV()
 32    >>> clf = ns.DeepClassifier(obj)
 33    >>> clf.fit(X_train, y_train)
 34    >>> print(clf.score(clf.predict(X_test), y_test))
 35    """
 36
 37    def __init__(
 38        self,
 39        obj,
 40        verbose=0,
 41        # Defining depth
 42        n_layers=3,
 43        # CustomClassifier attributes
 44        n_hidden_features=5,
 45        activation_name="relu",
 46        a=0.01,
 47        nodes_sim="sobol",
 48        bias=True,
 49        dropout=0,
 50        direct_link=True,
 51        n_clusters=2,
 52        cluster_encode=True,
 53        type_clust="kmeans",
 54        type_scaling=("std", "std", "std"),
 55        col_sample=1,
 56        row_sample=1,
 57        seed=123,
 58        backend="cpu",
 59    ):
 60        super().__init__(
 61            obj=obj,
 62            n_hidden_features=n_hidden_features,
 63            activation_name=activation_name,
 64            a=a,
 65            nodes_sim=nodes_sim,
 66            bias=bias,
 67            dropout=dropout,
 68            direct_link=direct_link,
 69            n_clusters=n_clusters,
 70            cluster_encode=cluster_encode,
 71            type_clust=type_clust,
 72            type_scaling=type_scaling,
 73            col_sample=col_sample,
 74            row_sample=row_sample,
 75            seed=seed,
 76            backend=backend,
 77        )
 78
 79        assert n_layers >= 2, "must have n_layers >= 2"
 80
 81        self.stacked_obj = obj
 82        self.verbose = verbose
 83        self.n_layers = n_layers
 84
 85    def fit(self, X, y):
 86        """Fit Classification algorithms to X and y.
 87        Parameters
 88        ----------
 89        X : array-like,
 90            Training vectors, where rows is the number of samples
 91            and columns is the number of features.
 92        y : array-like,
 93            Training vectors, where rows is the number of samples
 94            and columns is the number of features.
 95        Returns
 96        -------
 97        A fitted object
 98        """
 99
100        if isinstance(X, np.ndarray):
101            X = pd.DataFrame(X)
102
103        # init layer
104        self.stacked_obj = CustomClassifier(
105            obj=self.stacked_obj,
106            n_hidden_features=self.n_hidden_features,
107            activation_name=self.activation_name,
108            a=self.a,
109            nodes_sim=self.nodes_sim,
110            bias=self.bias,
111            dropout=self.dropout,
112            direct_link=self.direct_link,
113            n_clusters=self.n_clusters,
114            cluster_encode=self.cluster_encode,
115            type_clust=self.type_clust,
116            type_scaling=self.type_scaling,
117            col_sample=self.col_sample,
118            row_sample=self.row_sample,
119            seed=self.seed,
120            backend=self.backend,
121        )
122
123        self.stacked_obj.fit(X, y)
124
125        if self.verbose > 0:
126            iterator = tqdm(range(self.n_layers - 1))
127        else:
128            iterator = range(self.n_layers - 1)
129
130        for _ in iterator:
131            self.stacked_obj = deepcopy(
132                CustomClassifier(
133                    obj=self.stacked_obj,
134                    n_hidden_features=self.n_hidden_features,
135                    activation_name=self.activation_name,
136                    a=self.a,
137                    nodes_sim=self.nodes_sim,
138                    bias=self.bias,
139                    dropout=self.dropout,
140                    direct_link=self.direct_link,
141                    n_clusters=self.n_clusters,
142                    cluster_encode=self.cluster_encode,
143                    type_clust=self.type_clust,
144                    type_scaling=self.type_scaling,
145                    col_sample=self.col_sample,
146                    row_sample=self.row_sample,
147                    seed=self.seed,
148                    backend=self.backend,
149                )
150            )
151
152            # self.stacked_obj.fit(X, y)
153
154        self.stacked_obj.fit(X, y)
155
156        self.obj = deepcopy(self.stacked_obj)
157
158        return self.obj
159
160    def predict(self, X):
161        return self.obj.predict(X)
162
163    def predict_proba(self, X):
164        return self.obj.predict_proba(X)
165
166    def score(self, X, y, scoring=None):
167        return self.obj.score(X, y, scoring)

Deep Classifier

Parameters

verbose : int, optional (default=0) Monitor progress when fitting.

Examples

>>> import nnetsauce as ns
>>> from sklearn.datasets import load_breast_cancer
>>> from sklearn.model_selection import train_test_split
>>> from sklearn.linear_model import LogisticRegressionCV
>>> data = load_breast_cancer()
>>> X = data.data
>>> y= data.target
>>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
>>> obj = LogisticRegressionCV()
>>> clf = ns.DeepClassifier(obj)
>>> clf.fit(X_train, y_train)
>>> print(clf.score(clf.predict(X_test), y_test))
DeepClassifier( obj, verbose=0, n_layers=3, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
37    def __init__(
38        self,
39        obj,
40        verbose=0,
41        # Defining depth
42        n_layers=3,
43        # CustomClassifier attributes
44        n_hidden_features=5,
45        activation_name="relu",
46        a=0.01,
47        nodes_sim="sobol",
48        bias=True,
49        dropout=0,
50        direct_link=True,
51        n_clusters=2,
52        cluster_encode=True,
53        type_clust="kmeans",
54        type_scaling=("std", "std", "std"),
55        col_sample=1,
56        row_sample=1,
57        seed=123,
58        backend="cpu",
59    ):
60        super().__init__(
61            obj=obj,
62            n_hidden_features=n_hidden_features,
63            activation_name=activation_name,
64            a=a,
65            nodes_sim=nodes_sim,
66            bias=bias,
67            dropout=dropout,
68            direct_link=direct_link,
69            n_clusters=n_clusters,
70            cluster_encode=cluster_encode,
71            type_clust=type_clust,
72            type_scaling=type_scaling,
73            col_sample=col_sample,
74            row_sample=row_sample,
75            seed=seed,
76            backend=backend,
77        )
78
79        assert n_layers >= 2, "must have n_layers >= 2"
80
81        self.stacked_obj = obj
82        self.verbose = verbose
83        self.n_layers = n_layers
stacked_obj
verbose
n_layers
def fit(self, X, y):
 85    def fit(self, X, y):
 86        """Fit Classification algorithms to X and y.
 87        Parameters
 88        ----------
 89        X : array-like,
 90            Training vectors, where rows is the number of samples
 91            and columns is the number of features.
 92        y : array-like,
 93            Training vectors, where rows is the number of samples
 94            and columns is the number of features.
 95        Returns
 96        -------
 97        A fitted object
 98        """
 99
100        if isinstance(X, np.ndarray):
101            X = pd.DataFrame(X)
102
103        # init layer
104        self.stacked_obj = CustomClassifier(
105            obj=self.stacked_obj,
106            n_hidden_features=self.n_hidden_features,
107            activation_name=self.activation_name,
108            a=self.a,
109            nodes_sim=self.nodes_sim,
110            bias=self.bias,
111            dropout=self.dropout,
112            direct_link=self.direct_link,
113            n_clusters=self.n_clusters,
114            cluster_encode=self.cluster_encode,
115            type_clust=self.type_clust,
116            type_scaling=self.type_scaling,
117            col_sample=self.col_sample,
118            row_sample=self.row_sample,
119            seed=self.seed,
120            backend=self.backend,
121        )
122
123        self.stacked_obj.fit(X, y)
124
125        if self.verbose > 0:
126            iterator = tqdm(range(self.n_layers - 1))
127        else:
128            iterator = range(self.n_layers - 1)
129
130        for _ in iterator:
131            self.stacked_obj = deepcopy(
132                CustomClassifier(
133                    obj=self.stacked_obj,
134                    n_hidden_features=self.n_hidden_features,
135                    activation_name=self.activation_name,
136                    a=self.a,
137                    nodes_sim=self.nodes_sim,
138                    bias=self.bias,
139                    dropout=self.dropout,
140                    direct_link=self.direct_link,
141                    n_clusters=self.n_clusters,
142                    cluster_encode=self.cluster_encode,
143                    type_clust=self.type_clust,
144                    type_scaling=self.type_scaling,
145                    col_sample=self.col_sample,
146                    row_sample=self.row_sample,
147                    seed=self.seed,
148                    backend=self.backend,
149                )
150            )
151
152            # self.stacked_obj.fit(X, y)
153
154        self.stacked_obj.fit(X, y)
155
156        self.obj = deepcopy(self.stacked_obj)
157
158        return self.obj

Fit Classification algorithms to X and y.

Parameters

X : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y : array-like, Training vectors, where rows is the number of samples and columns is the number of features.

Returns

A fitted object

def predict(self, X):
160    def predict(self, X):
161        return self.obj.predict(X)

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X):
163    def predict_proba(self, X):
164        return self.obj.predict_proba(X)

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
def score(self, X, y, scoring=None):
166    def score(self, X, y, scoring=None):
167        return self.obj.score(X, y, scoring)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                     'neg_mean_squared_error', 'neg_mean_squared_log_error',                     'neg_median_absolute_error', 'r2')

**kwargs: 
    additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class DeepRegressor(nnetsauce.CustomRegressor, sklearn.base.RegressorMixin):
 12class DeepRegressor(CustomRegressor, RegressorMixin):
 13    """
 14    Deep Regressor
 15
 16    Parameters
 17    ----------
 18    verbose : int, optional (default=0)
 19        Monitor progress when fitting.
 20
 21    Examples
 22    --------
 23    >>> import nnetsauce as ns
 24    >>> from sklearn.datasets import load_diabetes
 25    >>> from sklearn.model_selection import train_test_split
 26    >>> from sklearn.linear_model import RidgeCV
 27    >>> data = load_diabetes()
 28    >>> X = data.data
 29    >>> y= data.target
 30    >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
 31    >>> obj = RidgeCV()
 32    >>> clf = ns.DeepRegressor(obj)
 33    >>> clf.fit(X_train, y_train)
 34    >>> print(clf.score(clf.predict(X_test), y_test))
 35    """
 36
 37    def __init__(
 38        self,
 39        obj,
 40        verbose=0,
 41        # Defining depth
 42        n_layers=3,
 43        # CustomRegressor attributes
 44        n_hidden_features=5,
 45        activation_name="relu",
 46        a=0.01,
 47        nodes_sim="sobol",
 48        bias=True,
 49        dropout=0,
 50        direct_link=True,
 51        n_clusters=2,
 52        cluster_encode=True,
 53        type_clust="kmeans",
 54        type_scaling=("std", "std", "std"),
 55        col_sample=1,
 56        row_sample=1,
 57        seed=123,
 58        backend="cpu",
 59    ):
 60        super().__init__(
 61            obj=obj,
 62            n_hidden_features=n_hidden_features,
 63            activation_name=activation_name,
 64            a=a,
 65            nodes_sim=nodes_sim,
 66            bias=bias,
 67            dropout=dropout,
 68            direct_link=direct_link,
 69            n_clusters=n_clusters,
 70            cluster_encode=cluster_encode,
 71            type_clust=type_clust,
 72            type_scaling=type_scaling,
 73            col_sample=col_sample,
 74            row_sample=row_sample,
 75            seed=seed,
 76            backend=backend,
 77        )
 78
 79        assert n_layers >= 2, "must have n_layers >= 2"
 80
 81        self.stacked_obj = obj
 82        self.verbose = verbose
 83        self.n_layers = n_layers
 84
 85    def fit(self, X, y):
 86        """Fit Regression algorithms to X and y.
 87        Parameters
 88        ----------
 89        X : array-like,
 90            Training vectors, where rows is the number of samples
 91            and columns is the number of features.
 92        y : array-like,
 93            Training vectors, where rows is the number of samples
 94            and columns is the number of features.
 95        Returns
 96        -------
 97        A fitted object
 98        """
 99
100        if isinstance(X, np.ndarray):
101            X = pd.DataFrame(X)
102
103        # init layer
104        self.stacked_obj = CustomRegressor(
105            obj=self.stacked_obj,
106            n_hidden_features=self.n_hidden_features,
107            activation_name=self.activation_name,
108            a=self.a,
109            nodes_sim=self.nodes_sim,
110            bias=self.bias,
111            dropout=self.dropout,
112            direct_link=self.direct_link,
113            n_clusters=self.n_clusters,
114            cluster_encode=self.cluster_encode,
115            type_clust=self.type_clust,
116            type_scaling=self.type_scaling,
117            col_sample=self.col_sample,
118            row_sample=self.row_sample,
119            seed=self.seed,
120            backend=self.backend,
121        )
122
123        # self.stacked_obj.fit(X, y)
124
125        if self.verbose > 0:
126            iterator = tqdm(range(self.n_layers - 1))
127        else:
128            iterator = range(self.n_layers - 1)
129
130        for _ in iterator:
131            self.stacked_obj = deepcopy(
132                CustomRegressor(
133                    obj=self.stacked_obj,
134                    n_hidden_features=self.n_hidden_features,
135                    activation_name=self.activation_name,
136                    a=self.a,
137                    nodes_sim=self.nodes_sim,
138                    bias=self.bias,
139                    dropout=self.dropout,
140                    direct_link=self.direct_link,
141                    n_clusters=self.n_clusters,
142                    cluster_encode=self.cluster_encode,
143                    type_clust=self.type_clust,
144                    type_scaling=self.type_scaling,
145                    col_sample=self.col_sample,
146                    row_sample=self.row_sample,
147                    seed=self.seed,
148                    backend=self.backend,
149                )
150            )
151
152            # self.stacked_obj.fit(X, y)
153
154        self.stacked_obj.fit(X, y)
155
156        self.obj = deepcopy(self.stacked_obj)
157
158        return self.obj
159
160    def predict(self, X):
161        return self.obj.predict(X)
162
163    def score(self, X, y, scoring=None):
164        return self.obj.score(X, y, scoring)

Deep Regressor

Parameters

verbose : int, optional (default=0) Monitor progress when fitting.

Examples

>>> import nnetsauce as ns
>>> from sklearn.datasets import load_diabetes
>>> from sklearn.model_selection import train_test_split
>>> from sklearn.linear_model import RidgeCV
>>> data = load_diabetes()
>>> X = data.data
>>> y= data.target
>>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
>>> obj = RidgeCV()
>>> clf = ns.DeepRegressor(obj)
>>> clf.fit(X_train, y_train)
>>> print(clf.score(clf.predict(X_test), y_test))
DeepRegressor( obj, verbose=0, n_layers=3, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
37    def __init__(
38        self,
39        obj,
40        verbose=0,
41        # Defining depth
42        n_layers=3,
43        # CustomRegressor attributes
44        n_hidden_features=5,
45        activation_name="relu",
46        a=0.01,
47        nodes_sim="sobol",
48        bias=True,
49        dropout=0,
50        direct_link=True,
51        n_clusters=2,
52        cluster_encode=True,
53        type_clust="kmeans",
54        type_scaling=("std", "std", "std"),
55        col_sample=1,
56        row_sample=1,
57        seed=123,
58        backend="cpu",
59    ):
60        super().__init__(
61            obj=obj,
62            n_hidden_features=n_hidden_features,
63            activation_name=activation_name,
64            a=a,
65            nodes_sim=nodes_sim,
66            bias=bias,
67            dropout=dropout,
68            direct_link=direct_link,
69            n_clusters=n_clusters,
70            cluster_encode=cluster_encode,
71            type_clust=type_clust,
72            type_scaling=type_scaling,
73            col_sample=col_sample,
74            row_sample=row_sample,
75            seed=seed,
76            backend=backend,
77        )
78
79        assert n_layers >= 2, "must have n_layers >= 2"
80
81        self.stacked_obj = obj
82        self.verbose = verbose
83        self.n_layers = n_layers
stacked_obj
verbose
n_layers
def fit(self, X, y):
 85    def fit(self, X, y):
 86        """Fit Regression algorithms to X and y.
 87        Parameters
 88        ----------
 89        X : array-like,
 90            Training vectors, where rows is the number of samples
 91            and columns is the number of features.
 92        y : array-like,
 93            Training vectors, where rows is the number of samples
 94            and columns is the number of features.
 95        Returns
 96        -------
 97        A fitted object
 98        """
 99
100        if isinstance(X, np.ndarray):
101            X = pd.DataFrame(X)
102
103        # init layer
104        self.stacked_obj = CustomRegressor(
105            obj=self.stacked_obj,
106            n_hidden_features=self.n_hidden_features,
107            activation_name=self.activation_name,
108            a=self.a,
109            nodes_sim=self.nodes_sim,
110            bias=self.bias,
111            dropout=self.dropout,
112            direct_link=self.direct_link,
113            n_clusters=self.n_clusters,
114            cluster_encode=self.cluster_encode,
115            type_clust=self.type_clust,
116            type_scaling=self.type_scaling,
117            col_sample=self.col_sample,
118            row_sample=self.row_sample,
119            seed=self.seed,
120            backend=self.backend,
121        )
122
123        # self.stacked_obj.fit(X, y)
124
125        if self.verbose > 0:
126            iterator = tqdm(range(self.n_layers - 1))
127        else:
128            iterator = range(self.n_layers - 1)
129
130        for _ in iterator:
131            self.stacked_obj = deepcopy(
132                CustomRegressor(
133                    obj=self.stacked_obj,
134                    n_hidden_features=self.n_hidden_features,
135                    activation_name=self.activation_name,
136                    a=self.a,
137                    nodes_sim=self.nodes_sim,
138                    bias=self.bias,
139                    dropout=self.dropout,
140                    direct_link=self.direct_link,
141                    n_clusters=self.n_clusters,
142                    cluster_encode=self.cluster_encode,
143                    type_clust=self.type_clust,
144                    type_scaling=self.type_scaling,
145                    col_sample=self.col_sample,
146                    row_sample=self.row_sample,
147                    seed=self.seed,
148                    backend=self.backend,
149                )
150            )
151
152            # self.stacked_obj.fit(X, y)
153
154        self.stacked_obj.fit(X, y)
155
156        self.obj = deepcopy(self.stacked_obj)
157
158        return self.obj

Fit Regression algorithms to X and y.

Parameters

X : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y : array-like, Training vectors, where rows is the number of samples and columns is the number of features.

Returns

A fitted object

def predict(self, X):
160    def predict(self, X):
161        return self.obj.predict(X)

Predict test data X.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

    **kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def score(self, X, y, scoring=None):
163    def score(self, X, y, scoring=None):
164        return self.obj.score(X, y, scoring)

Score the model on test set features X and response y.

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                     'neg_mean_squared_error', 'neg_mean_squared_log_error',                     'neg_median_absolute_error', 'r2')

**kwargs: 
    additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class DeepMTS(nnetsauce.MTS):
 10class DeepMTS(MTS):
 11    """Univariate and multivariate time series (DeepMTS) forecasting with Quasi-Randomized networks (Work in progress /!\)
 12
 13    Parameters:
 14
 15        obj: object.
 16            any object containing a method fit (obj.fit()) and a method predict
 17            (obj.predict()).
 18
 19        n_layers: int.
 20            number of layers in the neural network.
 21
 22        n_hidden_features: int.
 23            number of nodes in the hidden layer.
 24
 25        activation_name: str.
 26            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'.
 27
 28        a: float.
 29            hyperparameter for 'prelu' or 'elu' activation function.
 30
 31        nodes_sim: str.
 32            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 33            'uniform'.
 34
 35        bias: boolean.
 36            indicates if the hidden layer contains a bias term (True) or not
 37            (False).
 38
 39        dropout: float.
 40            regularization parameter; (random) percentage of nodes dropped out
 41            of the training.
 42
 43        direct_link: boolean.
 44            indicates if the original predictors are included (True) in model's fitting or not (False).
 45
 46        n_clusters: int.
 47            number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering).
 48
 49        cluster_encode: bool.
 50            defines how the variable containing clusters is treated (default is one-hot)
 51            if `False`, then labels are used, without one-hot encoding.
 52
 53        type_clust: str.
 54            type of clustering method: currently k-means ('kmeans') or Gaussian
 55            Mixture Model ('gmm').
 56
 57        type_scaling: a tuple of 3 strings.
 58            scaling methods for inputs, hidden layer, and clustering respectively
 59            (and when relevant).
 60            Currently available: standardization ('std') or MinMax scaling ('minmax').
 61
 62        lags: int.
 63            number of lags used for each time series.
 64
 65        replications: int.
 66            number of replications (if needed, for predictive simulation). Default is 'None'.
 67
 68        kernel: str.
 69            the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'.
 70
 71        agg: str.
 72            either "mean" or "median" for simulation of bootstrap aggregating
 73
 74        seed: int.
 75            reproducibility seed for nodes_sim=='uniform' or predictive simulation.
 76
 77        backend: str.
 78            "cpu" or "gpu" or "tpu".
 79
 80        verbose: int.
 81            0: not printing; 1: printing
 82
 83        show_progress: bool.
 84            True: progress bar when fitting each series; False: no progress bar when fitting each series
 85
 86    Attributes:
 87
 88        fit_objs_: dict
 89            objects adjusted to each individual time series
 90
 91        y_: {array-like}
 92            DeepMTS responses (most recent observations first)
 93
 94        X_: {array-like}
 95            DeepMTS lags
 96
 97        xreg_: {array-like}
 98            external regressors
 99
100        y_means_: dict
101            a dictionary of each series mean values
102
103        preds_: {array-like}
104            successive model predictions
105
106        preds_std_: {array-like}
107            standard deviation around the predictions
108
109        return_std_: boolean
110            return uncertainty or not (set in predict)
111
112        df_: data frame
113            the input data frame, in case a data.frame is provided to `fit`
114
115    Examples:
116
117    Example 1:
118
119    ```python
120    import nnetsauce as ns
121    import numpy as np
122    from sklearn import linear_model
123    np.random.seed(123)
124
125    M = np.random.rand(10, 3)
126    M[:,0] = 10*M[:,0]
127    M[:,2] = 25*M[:,2]
128    print(M)
129
130    # Adjust Bayesian Ridge
131    regr4 = linear_model.BayesianRidge()
132    obj_DeepMTS = ns.DeepMTS(regr4, lags = 1, n_hidden_features=5)
133    obj_DeepMTS.fit(M)
134    print(obj_DeepMTS.predict())
135
136    # with credible intervals
137    print(obj_DeepMTS.predict(return_std=True, level=80))
138
139    print(obj_DeepMTS.predict(return_std=True, level=95))
140    ```
141
142    Example 2:
143
144    ```python
145    import nnetsauce as ns
146    import numpy as np
147    from sklearn import linear_model
148
149    dataset = {
150    'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'],
151    'series1' : [34, 30, 35.6, 33.3, 38.1],
152    'series2' : [4, 5.5, 5.6, 6.3, 5.1],
153    'series3' : [100, 100.5, 100.6, 100.2, 100.1]}
154    df = pd.DataFrame(dataset).set_index('date')
155    print(df)
156
157    # Adjust Bayesian Ridge
158    regr5 = linear_model.BayesianRidge()
159    obj_DeepMTS = ns.DeepMTS(regr5, lags = 1, n_hidden_features=5)
160    obj_DeepMTS.fit(df)
161    print(obj_DeepMTS.predict())
162
163    # with credible intervals
164    print(obj_DeepMTS.predict(return_std=True, level=80))
165
166    print(obj_DeepMTS.predict(return_std=True, level=95))
167    ```
168    """
169
170    # construct the object -----
171
172    def __init__(
173        self,
174        obj,
175        n_layers=3,
176        n_hidden_features=5,
177        activation_name="relu",
178        a=0.01,
179        nodes_sim="sobol",
180        bias=True,
181        dropout=0,
182        direct_link=True,
183        n_clusters=2,
184        cluster_encode=True,
185        type_clust="kmeans",
186        type_scaling=("std", "std", "std"),
187        lags=1,
188        type_pi="kde",
189        replications=None,
190        kernel=None,
191        agg="mean",
192        seed=123,
193        backend="cpu",
194        verbose=0,
195        show_progress=True,
196    ):
197        assert int(lags) == lags, "parameter 'lags' should be an integer"
198        assert n_layers >= 2, "must have n_layers >= 2"
199        self.n_layers = int(n_layers)
200
201        self.obj = DeepRegressor(
202            obj=obj,
203            verbose=0,
204            n_layers=self.n_layers,
205            n_hidden_features=n_hidden_features,
206            activation_name=activation_name,
207            a=a,
208            nodes_sim=nodes_sim,
209            bias=bias,
210            dropout=dropout,
211            direct_link=direct_link,
212            n_clusters=n_clusters,
213            cluster_encode=cluster_encode,
214            type_clust=type_clust,
215            type_scaling=type_scaling,
216            seed=seed,
217            backend=backend,
218        )
219
220        super().__init__(
221            obj=self.obj,
222            n_hidden_features=n_hidden_features,
223            activation_name=activation_name,
224            a=a,
225            nodes_sim=nodes_sim,
226            bias=bias,
227            dropout=dropout,
228            direct_link=direct_link,
229            n_clusters=n_clusters,
230            cluster_encode=cluster_encode,
231            type_clust=type_clust,
232            type_scaling=type_scaling,
233            seed=seed,
234            type_pi=type_pi,
235            replications=replications,
236            kernel=kernel,
237            agg=agg,
238            backend=backend,
239            verbose=verbose,
240            show_progress=show_progress,
241        )

Univariate and multivariate time series (DeepMTS) forecasting with Quasi-Randomized networks (Work in progress /!)

Parameters:

obj: object.
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict()).

n_layers: int.
    number of layers in the neural network.

n_hidden_features: int.
    number of nodes in the hidden layer.

activation_name: str.
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'.

a: float.
    hyperparameter for 'prelu' or 'elu' activation function.

nodes_sim: str.
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'.

bias: boolean.
    indicates if the hidden layer contains a bias term (True) or not
    (False).

dropout: float.
    regularization parameter; (random) percentage of nodes dropped out
    of the training.

direct_link: boolean.
    indicates if the original predictors are included (True) in model's fitting or not (False).

n_clusters: int.
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering).

cluster_encode: bool.
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding.

type_clust: str.
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm').

type_scaling: a tuple of 3 strings.
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax').

lags: int.
    number of lags used for each time series.

replications: int.
    number of replications (if needed, for predictive simulation). Default is 'None'.

kernel: str.
    the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'.

agg: str.
    either "mean" or "median" for simulation of bootstrap aggregating

seed: int.
    reproducibility seed for nodes_sim=='uniform' or predictive simulation.

backend: str.
    "cpu" or "gpu" or "tpu".

verbose: int.
    0: not printing; 1: printing

show_progress: bool.
    True: progress bar when fitting each series; False: no progress bar when fitting each series

Attributes:

fit_objs_: dict
    objects adjusted to each individual time series

y_: {array-like}
    DeepMTS responses (most recent observations first)

X_: {array-like}
    DeepMTS lags

xreg_: {array-like}
    external regressors

y_means_: dict
    a dictionary of each series mean values

preds_: {array-like}
    successive model predictions

preds_std_: {array-like}
    standard deviation around the predictions

return_std_: boolean
    return uncertainty or not (set in predict)

df_: data frame
    the input data frame, in case a data.frame is provided to `fit`

Examples:

Example 1:

import nnetsauce as ns
import numpy as np
from sklearn import linear_model
np.random.seed(123)

M = np.random.rand(10, 3)
M[:,0] = 10*M[:,0]
M[:,2] = 25*M[:,2]
print(M)

# Adjust Bayesian Ridge
regr4 = linear_model.BayesianRidge()
obj_DeepMTS = ns.DeepMTS(regr4, lags = 1, n_hidden_features=5)
obj_DeepMTS.fit(M)
print(obj_DeepMTS.predict())

# with credible intervals
print(obj_DeepMTS.predict(return_std=True, level=80))

print(obj_DeepMTS.predict(return_std=True, level=95))

Example 2:

import nnetsauce as ns
import numpy as np
from sklearn import linear_model

dataset = {
'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'],
'series1' : [34, 30, 35.6, 33.3, 38.1],
'series2' : [4, 5.5, 5.6, 6.3, 5.1],
'series3' : [100, 100.5, 100.6, 100.2, 100.1]}
df = pd.DataFrame(dataset).set_index('date')
print(df)

# Adjust Bayesian Ridge
regr5 = linear_model.BayesianRidge()
obj_DeepMTS = ns.DeepMTS(regr5, lags = 1, n_hidden_features=5)
obj_DeepMTS.fit(df)
print(obj_DeepMTS.predict())

# with credible intervals
print(obj_DeepMTS.predict(return_std=True, level=80))

print(obj_DeepMTS.predict(return_std=True, level=95))
DeepMTS( obj, n_layers=3, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), lags=1, type_pi='kde', replications=None, kernel=None, agg='mean', seed=123, backend='cpu', verbose=0, show_progress=True)
172    def __init__(
173        self,
174        obj,
175        n_layers=3,
176        n_hidden_features=5,
177        activation_name="relu",
178        a=0.01,
179        nodes_sim="sobol",
180        bias=True,
181        dropout=0,
182        direct_link=True,
183        n_clusters=2,
184        cluster_encode=True,
185        type_clust="kmeans",
186        type_scaling=("std", "std", "std"),
187        lags=1,
188        type_pi="kde",
189        replications=None,
190        kernel=None,
191        agg="mean",
192        seed=123,
193        backend="cpu",
194        verbose=0,
195        show_progress=True,
196    ):
197        assert int(lags) == lags, "parameter 'lags' should be an integer"
198        assert n_layers >= 2, "must have n_layers >= 2"
199        self.n_layers = int(n_layers)
200
201        self.obj = DeepRegressor(
202            obj=obj,
203            verbose=0,
204            n_layers=self.n_layers,
205            n_hidden_features=n_hidden_features,
206            activation_name=activation_name,
207            a=a,
208            nodes_sim=nodes_sim,
209            bias=bias,
210            dropout=dropout,
211            direct_link=direct_link,
212            n_clusters=n_clusters,
213            cluster_encode=cluster_encode,
214            type_clust=type_clust,
215            type_scaling=type_scaling,
216            seed=seed,
217            backend=backend,
218        )
219
220        super().__init__(
221            obj=self.obj,
222            n_hidden_features=n_hidden_features,
223            activation_name=activation_name,
224            a=a,
225            nodes_sim=nodes_sim,
226            bias=bias,
227            dropout=dropout,
228            direct_link=direct_link,
229            n_clusters=n_clusters,
230            cluster_encode=cluster_encode,
231            type_clust=type_clust,
232            type_scaling=type_scaling,
233            seed=seed,
234            type_pi=type_pi,
235            replications=replications,
236            kernel=kernel,
237            agg=agg,
238            backend=backend,
239            verbose=verbose,
240            show_progress=show_progress,
241        )
n_layers
obj
def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_predict_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class GLMClassifier(nnetsauce.glm.glm.GLM, sklearn.base.ClassifierMixin):
 16class GLMClassifier(GLM, ClassifierMixin):
 17    """Generalized 'linear' models using quasi-randomized networks (classification)
 18
 19    Parameters:
 20
 21        n_hidden_features: int
 22            number of nodes in the hidden layer
 23
 24        lambda1: float
 25            regularization parameter for GLM coefficients on original features
 26
 27        alpha1: float
 28            controls compromize between l1 and l2 norm of GLM coefficients on original features
 29
 30        lambda2: float
 31            regularization parameter for GLM coefficients on nonlinear features
 32
 33        alpha2: float
 34            controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features
 35
 36        activation_name: str
 37            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 38
 39        a: float
 40            hyperparameter for 'prelu' or 'elu' activation function
 41
 42        nodes_sim: str
 43            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 44            'uniform'
 45
 46        bias: boolean
 47            indicates if the hidden layer contains a bias term (True) or not
 48            (False)
 49
 50        dropout: float
 51            regularization parameter; (random) percentage of nodes dropped out
 52            of the training
 53
 54        direct_link: boolean
 55            indicates if the original predictors are included (True) in model's
 56            fitting or not (False)
 57
 58        n_clusters: int
 59            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 60                no clustering)
 61
 62        cluster_encode: bool
 63            defines how the variable containing clusters is treated (default is one-hot)
 64            if `False`, then labels are used, without one-hot encoding
 65
 66        type_clust: str
 67            type of clustering method: currently k-means ('kmeans') or Gaussian
 68            Mixture Model ('gmm')
 69
 70        type_scaling: a tuple of 3 strings
 71            scaling methods for inputs, hidden layer, and clustering respectively
 72            (and when relevant).
 73            Currently available: standardization ('std') or MinMax scaling ('minmax')
 74
 75        optimizer: object
 76            optimizer, from class nnetsauce.Optimizer
 77
 78        seed: int
 79            reproducibility seed for nodes_sim=='uniform'
 80
 81    Attributes:
 82
 83        beta_: vector
 84            regression coefficients
 85
 86    Examples:
 87
 88    See [https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_classification.py)
 89
 90    """
 91
 92    # construct the object -----
 93
 94    def __init__(
 95        self,
 96        n_hidden_features=5,
 97        lambda1=0.01,
 98        alpha1=0.5,
 99        lambda2=0.01,
100        alpha2=0.5,
101        family="expit",
102        activation_name="relu",
103        a=0.01,
104        nodes_sim="sobol",
105        bias=True,
106        dropout=0,
107        direct_link=True,
108        n_clusters=2,
109        cluster_encode=True,
110        type_clust="kmeans",
111        type_scaling=("std", "std", "std"),
112        optimizer=Optimizer(),
113        seed=123,
114    ):
115        super().__init__(
116            n_hidden_features=n_hidden_features,
117            lambda1=lambda1,
118            alpha1=alpha1,
119            lambda2=lambda2,
120            alpha2=alpha2,
121            activation_name=activation_name,
122            a=a,
123            nodes_sim=nodes_sim,
124            bias=bias,
125            dropout=dropout,
126            direct_link=direct_link,
127            n_clusters=n_clusters,
128            cluster_encode=cluster_encode,
129            type_clust=type_clust,
130            type_scaling=type_scaling,
131            optimizer=optimizer,
132            seed=seed,
133        )
134
135        self.family = family
136
137    def logit_loss(self, Y, row_index, XB):
138        self.n_classes = Y.shape[1]  # len(np.unique(y))
139        # Y = mo.one_hot_encode2(y, self.n_classes)
140        # Y = self.optimizer.one_hot_encode(y, self.n_classes)
141
142        # max_double = 709.0 # only if softmax
143        # XB[XB > max_double] = max_double
144        XB[XB > 709.0] = 709.0
145
146        if row_index is None:
147            return -np.mean(np.sum(Y * XB, axis=1) - logsumexp(XB))
148
149        return -np.mean(np.sum(Y[row_index, :] * XB, axis=1) - logsumexp(XB))
150
151    def expit_erf_loss(self, Y, row_index, XB):
152        # self.n_classes = len(np.unique(y))
153        # Y = mo.one_hot_encode2(y, self.n_classes)
154        # Y = self.optimizer.one_hot_encode(y, self.n_classes)
155        self.n_classes = Y.shape[1]
156
157        if row_index is None:
158            return -np.mean(np.sum(Y * XB, axis=1) - logsumexp(XB))
159
160        return -np.mean(np.sum(Y[row_index, :] * XB, axis=1) - logsumexp(XB))
161
162    def loss_func(
163        self,
164        beta,
165        group_index,
166        X,
167        Y,
168        y,
169        row_index=None,
170        type_loss="logit",
171        **kwargs
172    ):
173        res = {
174            "logit": self.logit_loss,
175            "expit": self.expit_erf_loss,
176            "erf": self.expit_erf_loss,
177        }
178
179        if row_index is None:
180            row_index = range(len(y))
181            XB = self.compute_XB(
182                X,
183                beta=np.reshape(beta, (X.shape[1], self.n_classes), order="F"),
184            )
185
186            return res[type_loss](Y, row_index, XB) + self.compute_penalty(
187                group_index=group_index, beta=beta
188            )
189
190        XB = self.compute_XB(
191            X,
192            beta=np.reshape(beta, (X.shape[1], self.n_classes), order="F"),
193            row_index=row_index,
194        )
195
196        return res[type_loss](Y, row_index, XB) + self.compute_penalty(
197            group_index=group_index, beta=beta
198        )
199
200    def fit(self, X, y, **kwargs):
201        """Fit GLM model to training data (X, y).
202
203        Args:
204
205            X: {array-like}, shape = [n_samples, n_features]
206                Training vectors, where n_samples is the number
207                of samples and n_features is the number of features.
208
209            y: array-like, shape = [n_samples]
210                Target values.
211
212            **kwargs: additional parameters to be passed to
213                    self.cook_training_set or self.obj.fit
214
215        Returns:
216
217            self: object
218
219        """
220
221        assert mx.is_factor(
222            y
223        ), "y must contain only integers"  # change is_factor and subsampling everywhere
224
225        self.beta_ = None
226
227        n, p = X.shape
228
229        self.group_index = n * X.shape[1]
230
231        self.n_classes = len(np.unique(y))
232
233        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
234
235        # Y = mo.one_hot_encode2(output_y, self.n_classes)
236        Y = self.optimizer.one_hot_encode(output_y, self.n_classes)
237
238        # initialization
239        beta_ = np.linalg.lstsq(scaled_Z, Y, rcond=None)[0]
240
241        # optimization
242        # fit(self, loss_func, response, x0, **kwargs):
243        # loss_func(self, beta, group_index, X, y,
244        #          row_index=None, type_loss="gaussian",
245        #          **kwargs)
246        self.optimizer.fit(
247            self.loss_func,
248            response=y,
249            x0=beta_.flatten(order="F"),
250            group_index=self.group_index,
251            X=scaled_Z,
252            Y=Y,
253            y=y,
254            type_loss=self.family,
255        )
256
257        self.beta_ = self.optimizer.results[0]
258
259        return self
260
261    def predict(self, X, **kwargs):
262        """Predict test data X.
263
264        Args:
265
266            X: {array-like}, shape = [n_samples, n_features]
267                Training vectors, where n_samples is the number
268                of samples and n_features is the number of features.
269
270            **kwargs: additional parameters to be passed to
271                    self.cook_test_set
272
273        Returns:
274
275            model predictions: {array-like}
276
277        """
278
279        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
280
281    def predict_proba(self, X, **kwargs):
282        """Predict probabilities for test data X.
283
284        Args:
285
286            X: {array-like}, shape = [n_samples, n_features]
287                Training vectors, where n_samples is the number
288                of samples and n_features is the number of features.
289
290            **kwargs: additional parameters to be passed to
291                    self.cook_test_set
292
293        Returns:
294
295            probability estimates for test data: {array-like}
296
297        """
298        if len(X.shape) == 1:
299            n_features = X.shape[0]
300            new_X = mo.rbind(
301                X.reshape(1, n_features),
302                np.ones(n_features).reshape(1, n_features),
303            )
304
305            Z = self.cook_test_set(new_X, **kwargs)
306
307        else:
308            Z = self.cook_test_set(X, **kwargs)
309
310        ZB = mo.safe_sparse_dot(
311            Z,
312            self.beta_.reshape(
313                self.n_classes,
314                X.shape[1] + self.n_hidden_features + self.n_clusters,
315            ).T,
316        )
317
318        if self.family == "logit":
319            exp_ZB = np.exp(ZB)
320
321            return exp_ZB / exp_ZB.sum(axis=1)[:, None]
322
323        if self.family == "expit":
324            exp_ZB = expit(ZB)
325
326            return exp_ZB / exp_ZB.sum(axis=1)[:, None]
327
328        if self.family == "erf":
329            exp_ZB = 0.5 * (1 + erf(ZB))
330
331            return exp_ZB / exp_ZB.sum(axis=1)[:, None]
332
333    def score(self, X, y, scoring=None, **kwargs):
334        """ Score the model on test set features X and response y. 
335
336        Args:
337        
338            X: {array-like}, shape = [n_samples, n_features]
339                Training vectors, where n_samples is the number 
340                of samples and n_features is the number of features
341
342            y: array-like, shape = [n_samples]
343                Target values
344
345            scoring: str
346                must be in ('accuracy', 'average_precision', \
347                            'brier_score_loss', 'f1', 'f1_micro', \
348                            'f1_macro', 'f1_weighted',  'f1_samples', \
349                            'neg_log_loss', 'precision', 'recall', \
350                            'roc_auc')
351            
352            **kwargs: additional parameters to be passed to scoring functions
353               
354        Returns: 
355        
356            model scores: {array-like}
357
358        """
359
360        preds = self.predict(X)
361
362        if scoring is None:
363            scoring = "accuracy"
364
365        # check inputs
366        assert scoring in (
367            "accuracy",
368            "average_precision",
369            "brier_score_loss",
370            "f1",
371            "f1_micro",
372            "f1_macro",
373            "f1_weighted",
374            "f1_samples",
375            "neg_log_loss",
376            "precision",
377            "recall",
378            "roc_auc",
379        ), "'scoring' should be in ('accuracy', 'average_precision', \
380                           'brier_score_loss', 'f1', 'f1_micro', \
381                           'f1_macro', 'f1_weighted',  'f1_samples', \
382                           'neg_log_loss', 'precision', 'recall', \
383                           'roc_auc')"
384
385        scoring_options = {
386            "accuracy": skm2.accuracy_score,
387            "average_precision": skm2.average_precision_score,
388            "brier_score_loss": skm2.brier_score_loss,
389            "f1": skm2.f1_score,
390            "f1_micro": skm2.f1_score,
391            "f1_macro": skm2.f1_score,
392            "f1_weighted": skm2.f1_score,
393            "f1_samples": skm2.f1_score,
394            "neg_log_loss": skm2.log_loss,
395            "precision": skm2.precision_score,
396            "recall": skm2.recall_score,
397            "roc_auc": skm2.roc_auc_score,
398        }
399
400        return scoring_options[scoring](y, preds, **kwargs)

Generalized 'linear' models using quasi-randomized networks (classification)

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

lambda1: float
    regularization parameter for GLM coefficients on original features

alpha1: float
    controls compromize between l1 and l2 norm of GLM coefficients on original features

lambda2: float
    regularization parameter for GLM coefficients on nonlinear features

alpha2: float
    controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

optimizer: object
    optimizer, from class nnetsauce.Optimizer

seed: int
    reproducibility seed for nodes_sim=='uniform'

Attributes:

beta_: vector
    regression coefficients

Examples:

See https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_classification.py

GLMClassifier( n_hidden_features=5, lambda1=0.01, alpha1=0.5, lambda2=0.01, alpha2=0.5, family='expit', activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), optimizer=<Optimizer object>, seed=123)
 94    def __init__(
 95        self,
 96        n_hidden_features=5,
 97        lambda1=0.01,
 98        alpha1=0.5,
 99        lambda2=0.01,
100        alpha2=0.5,
101        family="expit",
102        activation_name="relu",
103        a=0.01,
104        nodes_sim="sobol",
105        bias=True,
106        dropout=0,
107        direct_link=True,
108        n_clusters=2,
109        cluster_encode=True,
110        type_clust="kmeans",
111        type_scaling=("std", "std", "std"),
112        optimizer=Optimizer(),
113        seed=123,
114    ):
115        super().__init__(
116            n_hidden_features=n_hidden_features,
117            lambda1=lambda1,
118            alpha1=alpha1,
119            lambda2=lambda2,
120            alpha2=alpha2,
121            activation_name=activation_name,
122            a=a,
123            nodes_sim=nodes_sim,
124            bias=bias,
125            dropout=dropout,
126            direct_link=direct_link,
127            n_clusters=n_clusters,
128            cluster_encode=cluster_encode,
129            type_clust=type_clust,
130            type_scaling=type_scaling,
131            optimizer=optimizer,
132            seed=seed,
133        )
134
135        self.family = family
family
def logit_loss(self, Y, row_index, XB):
137    def logit_loss(self, Y, row_index, XB):
138        self.n_classes = Y.shape[1]  # len(np.unique(y))
139        # Y = mo.one_hot_encode2(y, self.n_classes)
140        # Y = self.optimizer.one_hot_encode(y, self.n_classes)
141
142        # max_double = 709.0 # only if softmax
143        # XB[XB > max_double] = max_double
144        XB[XB > 709.0] = 709.0
145
146        if row_index is None:
147            return -np.mean(np.sum(Y * XB, axis=1) - logsumexp(XB))
148
149        return -np.mean(np.sum(Y[row_index, :] * XB, axis=1) - logsumexp(XB))
def expit_erf_loss(self, Y, row_index, XB):
151    def expit_erf_loss(self, Y, row_index, XB):
152        # self.n_classes = len(np.unique(y))
153        # Y = mo.one_hot_encode2(y, self.n_classes)
154        # Y = self.optimizer.one_hot_encode(y, self.n_classes)
155        self.n_classes = Y.shape[1]
156
157        if row_index is None:
158            return -np.mean(np.sum(Y * XB, axis=1) - logsumexp(XB))
159
160        return -np.mean(np.sum(Y[row_index, :] * XB, axis=1) - logsumexp(XB))
def loss_func( self, beta, group_index, X, Y, y, row_index=None, type_loss='logit', **kwargs):
162    def loss_func(
163        self,
164        beta,
165        group_index,
166        X,
167        Y,
168        y,
169        row_index=None,
170        type_loss="logit",
171        **kwargs
172    ):
173        res = {
174            "logit": self.logit_loss,
175            "expit": self.expit_erf_loss,
176            "erf": self.expit_erf_loss,
177        }
178
179        if row_index is None:
180            row_index = range(len(y))
181            XB = self.compute_XB(
182                X,
183                beta=np.reshape(beta, (X.shape[1], self.n_classes), order="F"),
184            )
185
186            return res[type_loss](Y, row_index, XB) + self.compute_penalty(
187                group_index=group_index, beta=beta
188            )
189
190        XB = self.compute_XB(
191            X,
192            beta=np.reshape(beta, (X.shape[1], self.n_classes), order="F"),
193            row_index=row_index,
194        )
195
196        return res[type_loss](Y, row_index, XB) + self.compute_penalty(
197            group_index=group_index, beta=beta
198        )
def fit(self, X, y, **kwargs):
200    def fit(self, X, y, **kwargs):
201        """Fit GLM model to training data (X, y).
202
203        Args:
204
205            X: {array-like}, shape = [n_samples, n_features]
206                Training vectors, where n_samples is the number
207                of samples and n_features is the number of features.
208
209            y: array-like, shape = [n_samples]
210                Target values.
211
212            **kwargs: additional parameters to be passed to
213                    self.cook_training_set or self.obj.fit
214
215        Returns:
216
217            self: object
218
219        """
220
221        assert mx.is_factor(
222            y
223        ), "y must contain only integers"  # change is_factor and subsampling everywhere
224
225        self.beta_ = None
226
227        n, p = X.shape
228
229        self.group_index = n * X.shape[1]
230
231        self.n_classes = len(np.unique(y))
232
233        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
234
235        # Y = mo.one_hot_encode2(output_y, self.n_classes)
236        Y = self.optimizer.one_hot_encode(output_y, self.n_classes)
237
238        # initialization
239        beta_ = np.linalg.lstsq(scaled_Z, Y, rcond=None)[0]
240
241        # optimization
242        # fit(self, loss_func, response, x0, **kwargs):
243        # loss_func(self, beta, group_index, X, y,
244        #          row_index=None, type_loss="gaussian",
245        #          **kwargs)
246        self.optimizer.fit(
247            self.loss_func,
248            response=y,
249            x0=beta_.flatten(order="F"),
250            group_index=self.group_index,
251            X=scaled_Z,
252            Y=Y,
253            y=y,
254            type_loss=self.family,
255        )
256
257        self.beta_ = self.optimizer.results[0]
258
259        return self

Fit GLM model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
261    def predict(self, X, **kwargs):
262        """Predict test data X.
263
264        Args:
265
266            X: {array-like}, shape = [n_samples, n_features]
267                Training vectors, where n_samples is the number
268                of samples and n_features is the number of features.
269
270            **kwargs: additional parameters to be passed to
271                    self.cook_test_set
272
273        Returns:
274
275            model predictions: {array-like}
276
277        """
278
279        return np.argmax(self.predict_proba(X, **kwargs), axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
281    def predict_proba(self, X, **kwargs):
282        """Predict probabilities for test data X.
283
284        Args:
285
286            X: {array-like}, shape = [n_samples, n_features]
287                Training vectors, where n_samples is the number
288                of samples and n_features is the number of features.
289
290            **kwargs: additional parameters to be passed to
291                    self.cook_test_set
292
293        Returns:
294
295            probability estimates for test data: {array-like}
296
297        """
298        if len(X.shape) == 1:
299            n_features = X.shape[0]
300            new_X = mo.rbind(
301                X.reshape(1, n_features),
302                np.ones(n_features).reshape(1, n_features),
303            )
304
305            Z = self.cook_test_set(new_X, **kwargs)
306
307        else:
308            Z = self.cook_test_set(X, **kwargs)
309
310        ZB = mo.safe_sparse_dot(
311            Z,
312            self.beta_.reshape(
313                self.n_classes,
314                X.shape[1] + self.n_hidden_features + self.n_clusters,
315            ).T,
316        )
317
318        if self.family == "logit":
319            exp_ZB = np.exp(ZB)
320
321            return exp_ZB / exp_ZB.sum(axis=1)[:, None]
322
323        if self.family == "expit":
324            exp_ZB = expit(ZB)
325
326            return exp_ZB / exp_ZB.sum(axis=1)[:, None]
327
328        if self.family == "erf":
329            exp_ZB = 0.5 * (1 + erf(ZB))
330
331            return exp_ZB / exp_ZB.sum(axis=1)[:, None]

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
def score(self, X, y, scoring=None, **kwargs):
333    def score(self, X, y, scoring=None, **kwargs):
334        """ Score the model on test set features X and response y. 
335
336        Args:
337        
338            X: {array-like}, shape = [n_samples, n_features]
339                Training vectors, where n_samples is the number 
340                of samples and n_features is the number of features
341
342            y: array-like, shape = [n_samples]
343                Target values
344
345            scoring: str
346                must be in ('accuracy', 'average_precision', \
347                            'brier_score_loss', 'f1', 'f1_micro', \
348                            'f1_macro', 'f1_weighted',  'f1_samples', \
349                            'neg_log_loss', 'precision', 'recall', \
350                            'roc_auc')
351            
352            **kwargs: additional parameters to be passed to scoring functions
353               
354        Returns: 
355        
356            model scores: {array-like}
357
358        """
359
360        preds = self.predict(X)
361
362        if scoring is None:
363            scoring = "accuracy"
364
365        # check inputs
366        assert scoring in (
367            "accuracy",
368            "average_precision",
369            "brier_score_loss",
370            "f1",
371            "f1_micro",
372            "f1_macro",
373            "f1_weighted",
374            "f1_samples",
375            "neg_log_loss",
376            "precision",
377            "recall",
378            "roc_auc",
379        ), "'scoring' should be in ('accuracy', 'average_precision', \
380                           'brier_score_loss', 'f1', 'f1_micro', \
381                           'f1_macro', 'f1_weighted',  'f1_samples', \
382                           'neg_log_loss', 'precision', 'recall', \
383                           'roc_auc')"
384
385        scoring_options = {
386            "accuracy": skm2.accuracy_score,
387            "average_precision": skm2.average_precision_score,
388            "brier_score_loss": skm2.brier_score_loss,
389            "f1": skm2.f1_score,
390            "f1_micro": skm2.f1_score,
391            "f1_macro": skm2.f1_score,
392            "f1_weighted": skm2.f1_score,
393            "f1_samples": skm2.f1_score,
394            "neg_log_loss": skm2.log_loss,
395            "precision": skm2.precision_score,
396            "recall": skm2.recall_score,
397            "roc_auc": skm2.roc_auc_score,
398        }
399
400        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('accuracy', 'average_precision',                             'brier_score_loss', 'f1', 'f1_micro',                             'f1_macro', 'f1_weighted',  'f1_samples',                             'neg_log_loss', 'precision', 'recall',                             'roc_auc')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.glm.glm.GLM
lambda1
alpha1
lambda2
alpha2
optimizer
beta_
compute_XB
compute_XB2
penalty
compute_penalty
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
class GLMRegressor(nnetsauce.glm.glm.GLM, sklearn.base.RegressorMixin):
 14class GLMRegressor(GLM, RegressorMixin):
 15    """Generalized 'linear' models using quasi-randomized networks (regression)
 16
 17    Attributes:
 18
 19        n_hidden_features: int
 20            number of nodes in the hidden layer
 21
 22        lambda1: float
 23            regularization parameter for GLM coefficients on original features
 24
 25        alpha1: float
 26            controls compromize between l1 and l2 norm of GLM coefficients on original features
 27
 28        lambda2: float
 29            regularization parameter for GLM coefficients on nonlinear features
 30
 31        alpha2: float
 32            controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features
 33
 34        family: str
 35            "gaussian", "laplace" or "poisson" (for now)
 36
 37        activation_name: str
 38            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 39
 40        a: float
 41            hyperparameter for 'prelu' or 'elu' activation function
 42
 43        nodes_sim: str
 44            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 45            'uniform'
 46
 47        bias: boolean
 48            indicates if the hidden layer contains a bias term (True) or not
 49            (False)
 50
 51        dropout: float
 52            regularization parameter; (random) percentage of nodes dropped out
 53            of the training
 54
 55        direct_link: boolean
 56            indicates if the original predictors are included (True) in model's
 57            fitting or not (False)
 58
 59        n_clusters: int
 60            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 61                no clustering)
 62
 63        cluster_encode: bool
 64            defines how the variable containing clusters is treated (default is one-hot)
 65            if `False`, then labels are used, without one-hot encoding
 66
 67        type_clust: str
 68            type of clustering method: currently k-means ('kmeans') or Gaussian
 69            Mixture Model ('gmm')
 70
 71        type_scaling: a tuple of 3 strings
 72            scaling methods for inputs, hidden layer, and clustering respectively
 73            (and when relevant).
 74            Currently available: standardization ('std') or MinMax scaling ('minmax')
 75
 76        optimizer: object
 77            optimizer, from class nnetsauce.utils.Optimizer
 78
 79        seed: int
 80            reproducibility seed for nodes_sim=='uniform'
 81
 82    Attributes:
 83
 84        beta_: vector
 85            regression coefficients
 86
 87    Examples:
 88
 89    See [https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_regression.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_regression.py)
 90
 91    """
 92
 93    # construct the object -----
 94
 95    def __init__(
 96        self,
 97        n_hidden_features=5,
 98        lambda1=0.01,
 99        alpha1=0.5,
100        lambda2=0.01,
101        alpha2=0.5,
102        family="gaussian",
103        activation_name="relu",
104        a=0.01,
105        nodes_sim="sobol",
106        bias=True,
107        dropout=0,
108        direct_link=True,
109        n_clusters=2,
110        cluster_encode=True,
111        type_clust="kmeans",
112        type_scaling=("std", "std", "std"),
113        optimizer=Optimizer(),
114        seed=123,
115    ):
116        super().__init__(
117            n_hidden_features=n_hidden_features,
118            lambda1=lambda1,
119            alpha1=alpha1,
120            lambda2=lambda2,
121            alpha2=alpha2,
122            activation_name=activation_name,
123            a=a,
124            nodes_sim=nodes_sim,
125            bias=bias,
126            dropout=dropout,
127            direct_link=direct_link,
128            n_clusters=n_clusters,
129            cluster_encode=cluster_encode,
130            type_clust=type_clust,
131            type_scaling=type_scaling,
132            optimizer=optimizer,
133            seed=seed,
134        )
135
136        self.family = family
137
138    def gaussian_loss(self, y, row_index, XB):
139        return 0.5 * np.mean(np.square(y[row_index] - XB))
140
141    def laplace_loss(self, y, row_index, XB):
142        return 0.5 * np.mean(np.abs(y[row_index] - XB))
143
144    def poisson_loss(self, y, row_index, XB):
145        return -np.mean(y[row_index] * XB - np.exp(XB))
146
147    def loss_func(
148        self,
149        beta,
150        group_index,
151        X,
152        y,
153        row_index=None,
154        type_loss="gaussian",
155        **kwargs
156    ):
157        res = {
158            "gaussian": self.gaussian_loss,
159            "laplace": self.laplace_loss,
160            "poisson": self.poisson_loss,
161        }
162
163        if row_index is None:
164            row_index = range(len(y))
165            XB = self.compute_XB(X, beta=beta)
166
167            return res[type_loss](y, row_index, XB) + self.compute_penalty(
168                group_index=group_index, beta=beta
169            )
170
171        XB = self.compute_XB(X, beta=beta, row_index=row_index)
172
173        return res[type_loss](y, row_index, XB) + self.compute_penalty(
174            group_index=group_index, beta=beta
175        )
176
177    def fit(self, X, y, **kwargs):
178        """Fit GLM model to training data (X, y).
179
180        Args:
181
182            X: {array-like}, shape = [n_samples, n_features]
183                Training vectors, where n_samples is the number
184                of samples and n_features is the number of features.
185
186            y: array-like, shape = [n_samples]
187                Target values.
188
189            **kwargs: additional parameters to be passed to
190                    self.cook_training_set or self.obj.fit
191
192        Returns:
193
194            self: object
195
196        """
197
198        self.beta_ = None
199
200        self.n_iter = 0
201
202        n, self.group_index = X.shape
203
204        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
205
206        n_Z = scaled_Z.shape[0]
207
208        # initialization
209        beta_ = np.linalg.lstsq(scaled_Z, centered_y, rcond=None)[0]
210
211        # optimization
212        # fit(self, loss_func, response, x0, **kwargs):
213        # loss_func(self, beta, group_index, X, y,
214        #          row_index=None, type_loss="gaussian",
215        #          **kwargs)
216        self.optimizer.fit(
217            self.loss_func,
218            response=centered_y,
219            x0=beta_,
220            group_index=self.group_index,
221            X=scaled_Z,
222            y=centered_y,
223            type_loss=self.family,
224            **kwargs
225        )
226
227        self.beta_ = self.optimizer.results[0]
228
229        return self
230
231    def predict(self, X, **kwargs):
232        """Predict test data X.
233
234        Args:
235
236            X: {array-like}, shape = [n_samples, n_features]
237                Training vectors, where n_samples is the number
238                of samples and n_features is the number of features.
239
240            **kwargs: additional parameters to be passed to
241                    self.cook_test_set
242
243        Returns:
244
245            model predictions: {array-like}
246
247        """
248
249        if len(X.shape) == 1:
250            n_features = X.shape[0]
251            new_X = mo.rbind(
252                X.reshape(1, n_features),
253                np.ones(n_features).reshape(1, n_features),
254            )
255
256            return (
257                self.y_mean_
258                + np.dot(self.cook_test_set(new_X, **kwargs), self.beta_)
259            )[0]
260
261        return self.y_mean_ + np.dot(
262            self.cook_test_set(X, **kwargs), self.beta_
263        )
264
265    def score(self, X, y, scoring=None, **kwargs):
266        """ Score the model on test set features X and response y. 
267
268        Args:
269        
270            X: {array-like}, shape = [n_samples, n_features]
271                Training vectors, where n_samples is the number 
272                of samples and n_features is the number of features
273
274            y: array-like, shape = [n_samples]
275                Target values
276
277            scoring: str
278                must be in ('explained_variance', 'neg_mean_absolute_error', \
279                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
280                            'neg_median_absolute_error', 'r2')
281            
282            **kwargs: additional parameters to be passed to scoring functions
283               
284        Returns: 
285        
286            model scores: {array-like}
287            
288        """
289
290        preds = self.predict(X)
291
292        if type(preds) == tuple:  # if there are std. devs in the predictions
293            preds = preds[0]
294
295        if scoring is None:
296            scoring = "neg_mean_squared_error"
297
298        # check inputs
299        assert scoring in (
300            "explained_variance",
301            "neg_mean_absolute_error",
302            "neg_mean_squared_error",
303            "neg_mean_squared_log_error",
304            "neg_median_absolute_error",
305            "r2",
306        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
307                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
308                           'neg_median_absolute_error', 'r2')"
309
310        scoring_options = {
311            "explained_variance": skm2.explained_variance_score,
312            "neg_mean_absolute_error": skm2.mean_absolute_error,
313            "neg_mean_squared_error": skm2.mean_squared_error,
314            "neg_mean_squared_log_error": skm2.mean_squared_log_error,
315            "neg_median_absolute_error": skm2.median_absolute_error,
316            "r2": skm2.r2_score,
317        }
318
319        return scoring_options[scoring](y, preds, **kwargs)

Generalized 'linear' models using quasi-randomized networks (regression)

Attributes:

n_hidden_features: int
    number of nodes in the hidden layer

lambda1: float
    regularization parameter for GLM coefficients on original features

alpha1: float
    controls compromize between l1 and l2 norm of GLM coefficients on original features

lambda2: float
    regularization parameter for GLM coefficients on nonlinear features

alpha2: float
    controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features

family: str
    "gaussian", "laplace" or "poisson" (for now)

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

optimizer: object
    optimizer, from class nnetsauce.utils.Optimizer

seed: int
    reproducibility seed for nodes_sim=='uniform'

Attributes:

beta_: vector
    regression coefficients

Examples:

See https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_regression.py

GLMRegressor( n_hidden_features=5, lambda1=0.01, alpha1=0.5, lambda2=0.01, alpha2=0.5, family='gaussian', activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), optimizer=<Optimizer object>, seed=123)
 95    def __init__(
 96        self,
 97        n_hidden_features=5,
 98        lambda1=0.01,
 99        alpha1=0.5,
100        lambda2=0.01,
101        alpha2=0.5,
102        family="gaussian",
103        activation_name="relu",
104        a=0.01,
105        nodes_sim="sobol",
106        bias=True,
107        dropout=0,
108        direct_link=True,
109        n_clusters=2,
110        cluster_encode=True,
111        type_clust="kmeans",
112        type_scaling=("std", "std", "std"),
113        optimizer=Optimizer(),
114        seed=123,
115    ):
116        super().__init__(
117            n_hidden_features=n_hidden_features,
118            lambda1=lambda1,
119            alpha1=alpha1,
120            lambda2=lambda2,
121            alpha2=alpha2,
122            activation_name=activation_name,
123            a=a,
124            nodes_sim=nodes_sim,
125            bias=bias,
126            dropout=dropout,
127            direct_link=direct_link,
128            n_clusters=n_clusters,
129            cluster_encode=cluster_encode,
130            type_clust=type_clust,
131            type_scaling=type_scaling,
132            optimizer=optimizer,
133            seed=seed,
134        )
135
136        self.family = family
family
def gaussian_loss(self, y, row_index, XB):
138    def gaussian_loss(self, y, row_index, XB):
139        return 0.5 * np.mean(np.square(y[row_index] - XB))
def laplace_loss(self, y, row_index, XB):
141    def laplace_loss(self, y, row_index, XB):
142        return 0.5 * np.mean(np.abs(y[row_index] - XB))
def poisson_loss(self, y, row_index, XB):
144    def poisson_loss(self, y, row_index, XB):
145        return -np.mean(y[row_index] * XB - np.exp(XB))
def loss_func( self, beta, group_index, X, y, row_index=None, type_loss='gaussian', **kwargs):
147    def loss_func(
148        self,
149        beta,
150        group_index,
151        X,
152        y,
153        row_index=None,
154        type_loss="gaussian",
155        **kwargs
156    ):
157        res = {
158            "gaussian": self.gaussian_loss,
159            "laplace": self.laplace_loss,
160            "poisson": self.poisson_loss,
161        }
162
163        if row_index is None:
164            row_index = range(len(y))
165            XB = self.compute_XB(X, beta=beta)
166
167            return res[type_loss](y, row_index, XB) + self.compute_penalty(
168                group_index=group_index, beta=beta
169            )
170
171        XB = self.compute_XB(X, beta=beta, row_index=row_index)
172
173        return res[type_loss](y, row_index, XB) + self.compute_penalty(
174            group_index=group_index, beta=beta
175        )
def fit(self, X, y, **kwargs):
177    def fit(self, X, y, **kwargs):
178        """Fit GLM model to training data (X, y).
179
180        Args:
181
182            X: {array-like}, shape = [n_samples, n_features]
183                Training vectors, where n_samples is the number
184                of samples and n_features is the number of features.
185
186            y: array-like, shape = [n_samples]
187                Target values.
188
189            **kwargs: additional parameters to be passed to
190                    self.cook_training_set or self.obj.fit
191
192        Returns:
193
194            self: object
195
196        """
197
198        self.beta_ = None
199
200        self.n_iter = 0
201
202        n, self.group_index = X.shape
203
204        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
205
206        n_Z = scaled_Z.shape[0]
207
208        # initialization
209        beta_ = np.linalg.lstsq(scaled_Z, centered_y, rcond=None)[0]
210
211        # optimization
212        # fit(self, loss_func, response, x0, **kwargs):
213        # loss_func(self, beta, group_index, X, y,
214        #          row_index=None, type_loss="gaussian",
215        #          **kwargs)
216        self.optimizer.fit(
217            self.loss_func,
218            response=centered_y,
219            x0=beta_,
220            group_index=self.group_index,
221            X=scaled_Z,
222            y=centered_y,
223            type_loss=self.family,
224            **kwargs
225        )
226
227        self.beta_ = self.optimizer.results[0]
228
229        return self

Fit GLM model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
231    def predict(self, X, **kwargs):
232        """Predict test data X.
233
234        Args:
235
236            X: {array-like}, shape = [n_samples, n_features]
237                Training vectors, where n_samples is the number
238                of samples and n_features is the number of features.
239
240            **kwargs: additional parameters to be passed to
241                    self.cook_test_set
242
243        Returns:
244
245            model predictions: {array-like}
246
247        """
248
249        if len(X.shape) == 1:
250            n_features = X.shape[0]
251            new_X = mo.rbind(
252                X.reshape(1, n_features),
253                np.ones(n_features).reshape(1, n_features),
254            )
255
256            return (
257                self.y_mean_
258                + np.dot(self.cook_test_set(new_X, **kwargs), self.beta_)
259            )[0]
260
261        return self.y_mean_ + np.dot(
262            self.cook_test_set(X, **kwargs), self.beta_
263        )

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def score(self, X, y, scoring=None, **kwargs):
265    def score(self, X, y, scoring=None, **kwargs):
266        """ Score the model on test set features X and response y. 
267
268        Args:
269        
270            X: {array-like}, shape = [n_samples, n_features]
271                Training vectors, where n_samples is the number 
272                of samples and n_features is the number of features
273
274            y: array-like, shape = [n_samples]
275                Target values
276
277            scoring: str
278                must be in ('explained_variance', 'neg_mean_absolute_error', \
279                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
280                            'neg_median_absolute_error', 'r2')
281            
282            **kwargs: additional parameters to be passed to scoring functions
283               
284        Returns: 
285        
286            model scores: {array-like}
287            
288        """
289
290        preds = self.predict(X)
291
292        if type(preds) == tuple:  # if there are std. devs in the predictions
293            preds = preds[0]
294
295        if scoring is None:
296            scoring = "neg_mean_squared_error"
297
298        # check inputs
299        assert scoring in (
300            "explained_variance",
301            "neg_mean_absolute_error",
302            "neg_mean_squared_error",
303            "neg_mean_squared_log_error",
304            "neg_median_absolute_error",
305            "r2",
306        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
307                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
308                           'neg_median_absolute_error', 'r2')"
309
310        scoring_options = {
311            "explained_variance": skm2.explained_variance_score,
312            "neg_mean_absolute_error": skm2.mean_absolute_error,
313            "neg_mean_squared_error": skm2.mean_squared_error,
314            "neg_mean_squared_log_error": skm2.mean_squared_log_error,
315            "neg_median_absolute_error": skm2.median_absolute_error,
316            "r2": skm2.r2_score,
317        }
318
319        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                             'neg_mean_squared_error', 'neg_mean_squared_log_error',                             'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.glm.glm.GLM
lambda1
alpha1
lambda2
alpha2
optimizer
beta_
compute_XB
compute_XB2
penalty
compute_penalty
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
class LazyClassifier(nnetsauce.custom.custom.Custom, sklearn.base.ClassifierMixin):
 84class LazyClassifier(Custom, ClassifierMixin):
 85    """
 86    This module helps in fitting to all the classification algorithms that are available in Scikit-learn to nnetsauce's CustomClassifier
 87    Parameters
 88    ----------
 89    verbose : int, optional (default=0)
 90        For the liblinear and lbfgs solvers set verbose to any positive
 91        number for verbosity.
 92    ignore_warnings : bool, optional (default=True)
 93        When set to True, the warning related to algorigms that are not able to run are ignored.
 94    custom_metric : function, optional (default=None)
 95        When function is provided, models are evaluated based on the custom evaluation metric provided.
 96    predictions : bool, optional (default=False)
 97        When set to True, the predictions of all the models models are returned as dataframe.
 98    classifiers : list, optional (default="all")
 99        When function is provided, trains the chosen classifier(s).
100    n_jobs : int, when possible, run in parallel
101
102    Examples
103    --------
104    >>> import nnetsauce as ns
105    >>> from sklearn.datasets import load_breast_cancer
106    >>> from sklearn.model_selection import train_test_split
107    >>> data = load_breast_cancer()
108    >>> X = data.data
109    >>> y= data.target
110    >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
111    >>> clf = ns.LazyClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
112    >>> models, predictions = clf.fit(X_train, X_test, y_train, y_test)
113    >>> model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
114    >>> print(models)
115    """
116
117    def __init__(
118        self,
119        verbose=0,
120        ignore_warnings=True,
121        custom_metric=None,
122        predictions=False,
123        random_state=42,
124        classifiers="all",
125        preprocess=False,
126        n_jobs=None,
127        # CustomClassifier attributes
128        obj=None,
129        n_hidden_features=5,
130        activation_name="relu",
131        a=0.01,
132        nodes_sim="sobol",
133        bias=True,
134        dropout=0,
135        direct_link=True,
136        n_clusters=2,
137        cluster_encode=True,
138        type_clust="kmeans",
139        type_scaling=("std", "std", "std"),
140        col_sample=1,
141        row_sample=1,
142        seed=123,
143        backend="cpu",
144    ):
145        self.verbose = verbose
146        self.ignore_warnings = ignore_warnings
147        self.custom_metric = custom_metric
148        self.predictions = predictions
149        self.models = {}
150        self.random_state = random_state
151        self.classifiers = classifiers
152        self.preprocess = preprocess
153        self.n_jobs = n_jobs
154        super().__init__(
155            obj=obj,
156            n_hidden_features=n_hidden_features,
157            activation_name=activation_name,
158            a=a,
159            nodes_sim=nodes_sim,
160            bias=bias,
161            dropout=dropout,
162            direct_link=direct_link,
163            n_clusters=n_clusters,
164            cluster_encode=cluster_encode,
165            type_clust=type_clust,
166            type_scaling=type_scaling,
167            col_sample=col_sample,
168            row_sample=row_sample,
169            seed=seed,
170            backend=backend,
171        )
172
173    def fit(self, X_train, X_test, y_train, y_test):
174        """Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test.
175        Parameters
176        ----------
177        X_train : array-like,
178            Training vectors, where rows is the number of samples
179            and columns is the number of features.
180        X_test : array-like,
181            Testing vectors, where rows is the number of samples
182            and columns is the number of features.
183        y_train : array-like,
184            Training vectors, where rows is the number of samples
185            and columns is the number of features.
186        y_test : array-like,
187            Testing vectors, where rows is the number of samples
188            and columns is the number of features.
189        Returns
190        -------
191        scores : Pandas DataFrame
192            Returns metrics of all the models in a Pandas DataFrame.
193        predictions : Pandas DataFrame
194            Returns predictions of all the models in a Pandas DataFrame.
195        """
196        Accuracy = []
197        B_Accuracy = []
198        ROC_AUC = []
199        F1 = []
200        names = []
201        TIME = []
202        predictions = {}
203
204        if self.custom_metric is not None:
205            CUSTOM_METRIC = []
206
207        if isinstance(X_train, np.ndarray):
208            X_train = pd.DataFrame(X_train)
209            X_test = pd.DataFrame(X_test)
210
211        numeric_features = X_train.select_dtypes(include=[np.number]).columns
212        categorical_features = X_train.select_dtypes(include=["object"]).columns
213
214        categorical_low, categorical_high = get_card_split(
215            X_train, categorical_features
216        )
217
218        if self.preprocess is True:
219            preprocessor = ColumnTransformer(
220                transformers=[
221                    ("numeric", numeric_transformer, numeric_features),
222                    (
223                        "categorical_low",
224                        categorical_transformer_low,
225                        categorical_low,
226                    ),
227                    (
228                        "categorical_high",
229                        categorical_transformer_high,
230                        categorical_high,
231                    ),
232                ]
233            )
234
235        if self.classifiers == "all":
236            self.classifiers = [
237                item
238                for sublist in [
239                    CLASSIFIERS,
240                    MULTITASKCLASSIFIERS,
241                    SIMPLEMULTITASKCLASSIFIERS,
242                ]
243                for item in sublist
244            ]
245
246        else:
247            try:
248                temp_list = []
249                for classifier in self.classifiers:
250                    full_name = (classifier.__name__, classifier)
251                    temp_list.append(full_name)
252                self.classifiers = temp_list
253            except Exception as exception:
254                print(exception)
255                print("Invalid Classifier(s)")
256
257        if self.preprocess is True:
258            for name, model in tqdm(self.classifiers):  # do parallel exec
259                other_args = (
260                    {}
261                )  # use this trick for `random_state` too --> refactor
262                try:
263                    if (
264                        "n_jobs" in model().get_params().keys()
265                        and name.find("LogisticRegression") == -1
266                    ):
267                        other_args["n_jobs"] = self.n_jobs
268                except Exception:
269                    pass
270
271                start = time.time()
272
273                try:
274                    if "random_state" in model().get_params().keys():
275                        pipe = Pipeline(
276                            [
277                                ("preprocessor", preprocessor),
278                                (
279                                    "classifier",
280                                    CustomClassifier(
281                                        obj=model(
282                                            random_state=self.random_state,
283                                            **other_args
284                                        ),
285                                        n_hidden_features=self.n_hidden_features,
286                                        activation_name=self.activation_name,
287                                        a=self.a,
288                                        nodes_sim=self.nodes_sim,
289                                        bias=self.bias,
290                                        dropout=self.dropout,
291                                        direct_link=self.direct_link,
292                                        n_clusters=self.n_clusters,
293                                        cluster_encode=self.cluster_encode,
294                                        type_clust=self.type_clust,
295                                        type_scaling=self.type_scaling,
296                                        col_sample=self.col_sample,
297                                        row_sample=self.row_sample,
298                                        seed=self.seed,
299                                        backend=self.backend,
300                                    ),
301                                ),
302                            ]
303                        )
304
305                    else:
306                        pipe = Pipeline(
307                            [
308                                ("preprocessor", preprocessor),
309                                (
310                                    "classifier",
311                                    CustomClassifier(
312                                        obj=model(**other_args),
313                                        n_hidden_features=self.n_hidden_features,
314                                        activation_name=self.activation_name,
315                                        a=self.a,
316                                        nodes_sim=self.nodes_sim,
317                                        bias=self.bias,
318                                        dropout=self.dropout,
319                                        direct_link=self.direct_link,
320                                        n_clusters=self.n_clusters,
321                                        cluster_encode=self.cluster_encode,
322                                        type_clust=self.type_clust,
323                                        type_scaling=self.type_scaling,
324                                        col_sample=self.col_sample,
325                                        row_sample=self.row_sample,
326                                        seed=self.seed,
327                                        backend=self.backend,
328                                    ),
329                                ),
330                            ]
331                        )
332
333                    pipe.fit(X_train, y_train)
334                    self.models[name] = pipe
335                    y_pred = pipe.predict(X_test)
336                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
337                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
338                    f1 = f1_score(y_test, y_pred, average="weighted")
339                    try:
340                        roc_auc = roc_auc_score(y_test, y_pred)
341                    except Exception as exception:
342                        roc_auc = None
343                        if self.ignore_warnings is False:
344                            print("ROC AUC couldn't be calculated for " + name)
345                            print(exception)
346                    names.append(name)
347                    Accuracy.append(accuracy)
348                    B_Accuracy.append(b_accuracy)
349                    ROC_AUC.append(roc_auc)
350                    F1.append(f1)
351                    TIME.append(time.time() - start)
352                    if self.custom_metric is not None:
353                        custom_metric = self.custom_metric(y_test, y_pred)
354                        CUSTOM_METRIC.append(custom_metric)
355                    if self.verbose > 0:
356                        if self.custom_metric is not None:
357                            print(
358                                {
359                                    "Model": name,
360                                    "Accuracy": accuracy,
361                                    "Balanced Accuracy": b_accuracy,
362                                    "ROC AUC": roc_auc,
363                                    "F1 Score": f1,
364                                    self.custom_metric.__name__: custom_metric,
365                                    "Time taken": time.time() - start,
366                                }
367                            )
368                        else:
369                            print(
370                                {
371                                    "Model": name,
372                                    "Accuracy": accuracy,
373                                    "Balanced Accuracy": b_accuracy,
374                                    "ROC AUC": roc_auc,
375                                    "F1 Score": f1,
376                                    "Time taken": time.time() - start,
377                                }
378                            )
379                    if self.predictions:
380                        predictions[name] = y_pred
381                except Exception as exception:
382                    if self.ignore_warnings is False:
383                        print(name + " model failed to execute")
384                        print(exception)
385                finally:
386                    continue
387
388        else:
389            for name, model in tqdm(self.classifiers):  # do parallel exec
390                other_args = (
391                    {}
392                )  # use this trick for `random_state` too --> refactor
393                try:
394                    if (
395                        "n_jobs" in model().get_params().keys()
396                        and name.find("LogisticRegression") == -1
397                    ):
398                        other_args["n_jobs"] = self.n_jobs
399                except Exception:
400                    pass
401
402                start = time.time()
403                try:
404                    if "random_state" in model().get_params().keys():
405                        pipe = CustomClassifier(
406                            obj=model(
407                                random_state=self.random_state, **other_args
408                            ),
409                            n_hidden_features=self.n_hidden_features,
410                            activation_name=self.activation_name,
411                            a=self.a,
412                            nodes_sim=self.nodes_sim,
413                            bias=self.bias,
414                            dropout=self.dropout,
415                            direct_link=self.direct_link,
416                            n_clusters=self.n_clusters,
417                            cluster_encode=self.cluster_encode,
418                            type_clust=self.type_clust,
419                            type_scaling=self.type_scaling,
420                            col_sample=self.col_sample,
421                            row_sample=self.row_sample,
422                            seed=self.seed,
423                            backend=self.backend,
424                        )
425                    else:
426                        pipe = CustomClassifier(
427                            obj=model(**other_args),
428                            n_hidden_features=self.n_hidden_features,
429                            activation_name=self.activation_name,
430                            a=self.a,
431                            nodes_sim=self.nodes_sim,
432                            bias=self.bias,
433                            dropout=self.dropout,
434                            direct_link=self.direct_link,
435                            n_clusters=self.n_clusters,
436                            cluster_encode=self.cluster_encode,
437                            type_clust=self.type_clust,
438                            type_scaling=self.type_scaling,
439                            col_sample=self.col_sample,
440                            row_sample=self.row_sample,
441                            seed=self.seed,
442                            backend=self.backend,
443                        )
444
445                    pipe.fit(X_train, y_train)
446                    self.models[name] = pipe
447                    y_pred = pipe.predict(X_test)
448                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
449                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
450                    f1 = f1_score(y_test, y_pred, average="weighted")
451                    try:
452                        roc_auc = roc_auc_score(y_test, y_pred)
453                    except Exception as exception:
454                        roc_auc = None
455                        if self.ignore_warnings is False:
456                            print("ROC AUC couldn't be calculated for " + name)
457                            print(exception)
458                    names.append(name)
459                    Accuracy.append(accuracy)
460                    B_Accuracy.append(b_accuracy)
461                    ROC_AUC.append(roc_auc)
462                    F1.append(f1)
463                    TIME.append(time.time() - start)
464                    if self.custom_metric is not None:
465                        custom_metric = self.custom_metric(y_test, y_pred)
466                        CUSTOM_METRIC.append(custom_metric)
467                    if self.verbose > 0:
468                        if self.custom_metric is not None:
469                            print(
470                                {
471                                    "Model": name,
472                                    "Accuracy": accuracy,
473                                    "Balanced Accuracy": b_accuracy,
474                                    "ROC AUC": roc_auc,
475                                    "F1 Score": f1,
476                                    self.custom_metric.__name__: custom_metric,
477                                    "Time taken": time.time() - start,
478                                }
479                            )
480                        else:
481                            print(
482                                {
483                                    "Model": name,
484                                    "Accuracy": accuracy,
485                                    "Balanced Accuracy": b_accuracy,
486                                    "ROC AUC": roc_auc,
487                                    "F1 Score": f1,
488                                    "Time taken": time.time() - start,
489                                }
490                            )
491                    if self.predictions:
492                        predictions[name] = y_pred
493                except Exception as exception:
494                    if self.ignore_warnings is False:
495                        print(name + " model failed to execute")
496                        print(exception)
497                finally:
498                    continue
499
500        if self.custom_metric is None:
501            scores = pd.DataFrame(
502                {
503                    "Model": names,
504                    "Accuracy": Accuracy,
505                    "Balanced Accuracy": B_Accuracy,
506                    "ROC AUC": ROC_AUC,
507                    "F1 Score": F1,
508                    "Time Taken": TIME,
509                }
510            )
511        else:
512            scores = pd.DataFrame(
513                {
514                    "Model": names,
515                    "Accuracy": Accuracy,
516                    "Balanced Accuracy": B_Accuracy,
517                    "ROC AUC": ROC_AUC,
518                    "F1 Score": F1,
519                    self.custom_metric.__name__: CUSTOM_METRIC,
520                    "Time Taken": TIME,
521                }
522            )
523        scores = scores.sort_values(by="Accuracy", ascending=False).set_index(
524            "Model"
525        )
526
527        if self.predictions:
528            predictions_df = pd.DataFrame.from_dict(predictions)
529        return scores, predictions_df if self.predictions is True else scores
530
531    def provide_models(self, X_train, X_test, y_train, y_test):
532        """
533        This function returns all the model objects trained in fit function.
534        If fit is not called already, then we call fit and then return the models.
535        Parameters
536        ----------
537        X_train : array-like,
538            Training vectors, where rows is the number of samples
539            and columns is the number of features.
540        X_test : array-like,
541            Testing vectors, where rows is the number of samples
542            and columns is the number of features.
543        y_train : array-like,
544            Training vectors, where rows is the number of samples
545            and columns is the number of features.
546        y_test : array-like,
547            Testing vectors, where rows is the number of samples
548            and columns is the number of features.
549        Returns
550        -------
551        models: dict-object,
552            Returns a dictionary with each model pipeline as value
553            with key as name of models.
554        """
555        if len(self.models.keys()) == 0:
556            self.fit(X_train, X_test, y_train, y_test)
557
558        return self.models

This module helps in fitting to all the classification algorithms that are available in Scikit-learn to nnetsauce's CustomClassifier

Parameters

verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. predictions : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. classifiers : list, optional (default="all") When function is provided, trains the chosen classifier(s). n_jobs : int, when possible, run in parallel

Examples

>>> import nnetsauce as ns
>>> from sklearn.datasets import load_breast_cancer
>>> from sklearn.model_selection import train_test_split
>>> data = load_breast_cancer()
>>> X = data.data
>>> y= data.target
>>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
>>> clf = ns.LazyClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
>>> models, predictions = clf.fit(X_train, X_test, y_train, y_test)
>>> model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
>>> print(models)
LazyClassifier( verbose=0, ignore_warnings=True, custom_metric=None, predictions=False, random_state=42, classifiers='all', preprocess=False, n_jobs=None, obj=None, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
117    def __init__(
118        self,
119        verbose=0,
120        ignore_warnings=True,
121        custom_metric=None,
122        predictions=False,
123        random_state=42,
124        classifiers="all",
125        preprocess=False,
126        n_jobs=None,
127        # CustomClassifier attributes
128        obj=None,
129        n_hidden_features=5,
130        activation_name="relu",
131        a=0.01,
132        nodes_sim="sobol",
133        bias=True,
134        dropout=0,
135        direct_link=True,
136        n_clusters=2,
137        cluster_encode=True,
138        type_clust="kmeans",
139        type_scaling=("std", "std", "std"),
140        col_sample=1,
141        row_sample=1,
142        seed=123,
143        backend="cpu",
144    ):
145        self.verbose = verbose
146        self.ignore_warnings = ignore_warnings
147        self.custom_metric = custom_metric
148        self.predictions = predictions
149        self.models = {}
150        self.random_state = random_state
151        self.classifiers = classifiers
152        self.preprocess = preprocess
153        self.n_jobs = n_jobs
154        super().__init__(
155            obj=obj,
156            n_hidden_features=n_hidden_features,
157            activation_name=activation_name,
158            a=a,
159            nodes_sim=nodes_sim,
160            bias=bias,
161            dropout=dropout,
162            direct_link=direct_link,
163            n_clusters=n_clusters,
164            cluster_encode=cluster_encode,
165            type_clust=type_clust,
166            type_scaling=type_scaling,
167            col_sample=col_sample,
168            row_sample=row_sample,
169            seed=seed,
170            backend=backend,
171        )
verbose
ignore_warnings
custom_metric
predictions
models
random_state
classifiers
preprocess
n_jobs
def fit(self, X_train, X_test, y_train, y_test):
173    def fit(self, X_train, X_test, y_train, y_test):
174        """Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test.
175        Parameters
176        ----------
177        X_train : array-like,
178            Training vectors, where rows is the number of samples
179            and columns is the number of features.
180        X_test : array-like,
181            Testing vectors, where rows is the number of samples
182            and columns is the number of features.
183        y_train : array-like,
184            Training vectors, where rows is the number of samples
185            and columns is the number of features.
186        y_test : array-like,
187            Testing vectors, where rows is the number of samples
188            and columns is the number of features.
189        Returns
190        -------
191        scores : Pandas DataFrame
192            Returns metrics of all the models in a Pandas DataFrame.
193        predictions : Pandas DataFrame
194            Returns predictions of all the models in a Pandas DataFrame.
195        """
196        Accuracy = []
197        B_Accuracy = []
198        ROC_AUC = []
199        F1 = []
200        names = []
201        TIME = []
202        predictions = {}
203
204        if self.custom_metric is not None:
205            CUSTOM_METRIC = []
206
207        if isinstance(X_train, np.ndarray):
208            X_train = pd.DataFrame(X_train)
209            X_test = pd.DataFrame(X_test)
210
211        numeric_features = X_train.select_dtypes(include=[np.number]).columns
212        categorical_features = X_train.select_dtypes(include=["object"]).columns
213
214        categorical_low, categorical_high = get_card_split(
215            X_train, categorical_features
216        )
217
218        if self.preprocess is True:
219            preprocessor = ColumnTransformer(
220                transformers=[
221                    ("numeric", numeric_transformer, numeric_features),
222                    (
223                        "categorical_low",
224                        categorical_transformer_low,
225                        categorical_low,
226                    ),
227                    (
228                        "categorical_high",
229                        categorical_transformer_high,
230                        categorical_high,
231                    ),
232                ]
233            )
234
235        if self.classifiers == "all":
236            self.classifiers = [
237                item
238                for sublist in [
239                    CLASSIFIERS,
240                    MULTITASKCLASSIFIERS,
241                    SIMPLEMULTITASKCLASSIFIERS,
242                ]
243                for item in sublist
244            ]
245
246        else:
247            try:
248                temp_list = []
249                for classifier in self.classifiers:
250                    full_name = (classifier.__name__, classifier)
251                    temp_list.append(full_name)
252                self.classifiers = temp_list
253            except Exception as exception:
254                print(exception)
255                print("Invalid Classifier(s)")
256
257        if self.preprocess is True:
258            for name, model in tqdm(self.classifiers):  # do parallel exec
259                other_args = (
260                    {}
261                )  # use this trick for `random_state` too --> refactor
262                try:
263                    if (
264                        "n_jobs" in model().get_params().keys()
265                        and name.find("LogisticRegression") == -1
266                    ):
267                        other_args["n_jobs"] = self.n_jobs
268                except Exception:
269                    pass
270
271                start = time.time()
272
273                try:
274                    if "random_state" in model().get_params().keys():
275                        pipe = Pipeline(
276                            [
277                                ("preprocessor", preprocessor),
278                                (
279                                    "classifier",
280                                    CustomClassifier(
281                                        obj=model(
282                                            random_state=self.random_state,
283                                            **other_args
284                                        ),
285                                        n_hidden_features=self.n_hidden_features,
286                                        activation_name=self.activation_name,
287                                        a=self.a,
288                                        nodes_sim=self.nodes_sim,
289                                        bias=self.bias,
290                                        dropout=self.dropout,
291                                        direct_link=self.direct_link,
292                                        n_clusters=self.n_clusters,
293                                        cluster_encode=self.cluster_encode,
294                                        type_clust=self.type_clust,
295                                        type_scaling=self.type_scaling,
296                                        col_sample=self.col_sample,
297                                        row_sample=self.row_sample,
298                                        seed=self.seed,
299                                        backend=self.backend,
300                                    ),
301                                ),
302                            ]
303                        )
304
305                    else:
306                        pipe = Pipeline(
307                            [
308                                ("preprocessor", preprocessor),
309                                (
310                                    "classifier",
311                                    CustomClassifier(
312                                        obj=model(**other_args),
313                                        n_hidden_features=self.n_hidden_features,
314                                        activation_name=self.activation_name,
315                                        a=self.a,
316                                        nodes_sim=self.nodes_sim,
317                                        bias=self.bias,
318                                        dropout=self.dropout,
319                                        direct_link=self.direct_link,
320                                        n_clusters=self.n_clusters,
321                                        cluster_encode=self.cluster_encode,
322                                        type_clust=self.type_clust,
323                                        type_scaling=self.type_scaling,
324                                        col_sample=self.col_sample,
325                                        row_sample=self.row_sample,
326                                        seed=self.seed,
327                                        backend=self.backend,
328                                    ),
329                                ),
330                            ]
331                        )
332
333                    pipe.fit(X_train, y_train)
334                    self.models[name] = pipe
335                    y_pred = pipe.predict(X_test)
336                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
337                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
338                    f1 = f1_score(y_test, y_pred, average="weighted")
339                    try:
340                        roc_auc = roc_auc_score(y_test, y_pred)
341                    except Exception as exception:
342                        roc_auc = None
343                        if self.ignore_warnings is False:
344                            print("ROC AUC couldn't be calculated for " + name)
345                            print(exception)
346                    names.append(name)
347                    Accuracy.append(accuracy)
348                    B_Accuracy.append(b_accuracy)
349                    ROC_AUC.append(roc_auc)
350                    F1.append(f1)
351                    TIME.append(time.time() - start)
352                    if self.custom_metric is not None:
353                        custom_metric = self.custom_metric(y_test, y_pred)
354                        CUSTOM_METRIC.append(custom_metric)
355                    if self.verbose > 0:
356                        if self.custom_metric is not None:
357                            print(
358                                {
359                                    "Model": name,
360                                    "Accuracy": accuracy,
361                                    "Balanced Accuracy": b_accuracy,
362                                    "ROC AUC": roc_auc,
363                                    "F1 Score": f1,
364                                    self.custom_metric.__name__: custom_metric,
365                                    "Time taken": time.time() - start,
366                                }
367                            )
368                        else:
369                            print(
370                                {
371                                    "Model": name,
372                                    "Accuracy": accuracy,
373                                    "Balanced Accuracy": b_accuracy,
374                                    "ROC AUC": roc_auc,
375                                    "F1 Score": f1,
376                                    "Time taken": time.time() - start,
377                                }
378                            )
379                    if self.predictions:
380                        predictions[name] = y_pred
381                except Exception as exception:
382                    if self.ignore_warnings is False:
383                        print(name + " model failed to execute")
384                        print(exception)
385                finally:
386                    continue
387
388        else:
389            for name, model in tqdm(self.classifiers):  # do parallel exec
390                other_args = (
391                    {}
392                )  # use this trick for `random_state` too --> refactor
393                try:
394                    if (
395                        "n_jobs" in model().get_params().keys()
396                        and name.find("LogisticRegression") == -1
397                    ):
398                        other_args["n_jobs"] = self.n_jobs
399                except Exception:
400                    pass
401
402                start = time.time()
403                try:
404                    if "random_state" in model().get_params().keys():
405                        pipe = CustomClassifier(
406                            obj=model(
407                                random_state=self.random_state, **other_args
408                            ),
409                            n_hidden_features=self.n_hidden_features,
410                            activation_name=self.activation_name,
411                            a=self.a,
412                            nodes_sim=self.nodes_sim,
413                            bias=self.bias,
414                            dropout=self.dropout,
415                            direct_link=self.direct_link,
416                            n_clusters=self.n_clusters,
417                            cluster_encode=self.cluster_encode,
418                            type_clust=self.type_clust,
419                            type_scaling=self.type_scaling,
420                            col_sample=self.col_sample,
421                            row_sample=self.row_sample,
422                            seed=self.seed,
423                            backend=self.backend,
424                        )
425                    else:
426                        pipe = CustomClassifier(
427                            obj=model(**other_args),
428                            n_hidden_features=self.n_hidden_features,
429                            activation_name=self.activation_name,
430                            a=self.a,
431                            nodes_sim=self.nodes_sim,
432                            bias=self.bias,
433                            dropout=self.dropout,
434                            direct_link=self.direct_link,
435                            n_clusters=self.n_clusters,
436                            cluster_encode=self.cluster_encode,
437                            type_clust=self.type_clust,
438                            type_scaling=self.type_scaling,
439                            col_sample=self.col_sample,
440                            row_sample=self.row_sample,
441                            seed=self.seed,
442                            backend=self.backend,
443                        )
444
445                    pipe.fit(X_train, y_train)
446                    self.models[name] = pipe
447                    y_pred = pipe.predict(X_test)
448                    accuracy = accuracy_score(y_test, y_pred, normalize=True)
449                    b_accuracy = balanced_accuracy_score(y_test, y_pred)
450                    f1 = f1_score(y_test, y_pred, average="weighted")
451                    try:
452                        roc_auc = roc_auc_score(y_test, y_pred)
453                    except Exception as exception:
454                        roc_auc = None
455                        if self.ignore_warnings is False:
456                            print("ROC AUC couldn't be calculated for " + name)
457                            print(exception)
458                    names.append(name)
459                    Accuracy.append(accuracy)
460                    B_Accuracy.append(b_accuracy)
461                    ROC_AUC.append(roc_auc)
462                    F1.append(f1)
463                    TIME.append(time.time() - start)
464                    if self.custom_metric is not None:
465                        custom_metric = self.custom_metric(y_test, y_pred)
466                        CUSTOM_METRIC.append(custom_metric)
467                    if self.verbose > 0:
468                        if self.custom_metric is not None:
469                            print(
470                                {
471                                    "Model": name,
472                                    "Accuracy": accuracy,
473                                    "Balanced Accuracy": b_accuracy,
474                                    "ROC AUC": roc_auc,
475                                    "F1 Score": f1,
476                                    self.custom_metric.__name__: custom_metric,
477                                    "Time taken": time.time() - start,
478                                }
479                            )
480                        else:
481                            print(
482                                {
483                                    "Model": name,
484                                    "Accuracy": accuracy,
485                                    "Balanced Accuracy": b_accuracy,
486                                    "ROC AUC": roc_auc,
487                                    "F1 Score": f1,
488                                    "Time taken": time.time() - start,
489                                }
490                            )
491                    if self.predictions:
492                        predictions[name] = y_pred
493                except Exception as exception:
494                    if self.ignore_warnings is False:
495                        print(name + " model failed to execute")
496                        print(exception)
497                finally:
498                    continue
499
500        if self.custom_metric is None:
501            scores = pd.DataFrame(
502                {
503                    "Model": names,
504                    "Accuracy": Accuracy,
505                    "Balanced Accuracy": B_Accuracy,
506                    "ROC AUC": ROC_AUC,
507                    "F1 Score": F1,
508                    "Time Taken": TIME,
509                }
510            )
511        else:
512            scores = pd.DataFrame(
513                {
514                    "Model": names,
515                    "Accuracy": Accuracy,
516                    "Balanced Accuracy": B_Accuracy,
517                    "ROC AUC": ROC_AUC,
518                    "F1 Score": F1,
519                    self.custom_metric.__name__: CUSTOM_METRIC,
520                    "Time Taken": TIME,
521                }
522            )
523        scores = scores.sort_values(by="Accuracy", ascending=False).set_index(
524            "Model"
525        )
526
527        if self.predictions:
528            predictions_df = pd.DataFrame.from_dict(predictions)
529        return scores, predictions_df if self.predictions is True else scores

Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.

def provide_models(self, X_train, X_test, y_train, y_test):
531    def provide_models(self, X_train, X_test, y_train, y_test):
532        """
533        This function returns all the model objects trained in fit function.
534        If fit is not called already, then we call fit and then return the models.
535        Parameters
536        ----------
537        X_train : array-like,
538            Training vectors, where rows is the number of samples
539            and columns is the number of features.
540        X_test : array-like,
541            Testing vectors, where rows is the number of samples
542            and columns is the number of features.
543        y_train : array-like,
544            Training vectors, where rows is the number of samples
545            and columns is the number of features.
546        y_test : array-like,
547            Testing vectors, where rows is the number of samples
548            and columns is the number of features.
549        Returns
550        -------
551        models: dict-object,
552            Returns a dictionary with each model pipeline as value
553            with key as name of models.
554        """
555        if len(self.models.keys()) == 0:
556            self.fit(X_train, X_test, y_train, y_test)
557
558        return self.models

This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.

def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.custom.custom.Custom
obj
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
sklearn.base.ClassifierMixin
score
class LazyRegressor(nnetsauce.custom.custom.Custom, sklearn.base.RegressorMixin):
 84class LazyRegressor(Custom, RegressorMixin):
 85    """
 86    This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's CustomRegressor
 87    Parameters
 88    ----------
 89    verbose : int, optional (default=0)
 90        For the liblinear and lbfgs solvers set verbose to any positive
 91        number for verbosity.
 92    ignore_warnings : bool, optional (default=True)
 93        When set to True, the warning related to algorigms that are not able to run are ignored.
 94    custom_metric : function, optional (default=None)
 95        When function is provided, models are evaluated based on the custom evaluation metric provided.
 96    prediction : bool, optional (default=False)
 97        When set to True, the predictions of all the models models are returned as dataframe.
 98    regressors : list, optional (default="all")
 99        When function is provided, trains the chosen regressor(s).
100    n_jobs : int, when possible, run in parallel
101
102    Examples
103    --------
104    >>> from lazypredict.Supervised import LazyRegressor
105    >>> from sklearn import datasets
106    >>> from sklearn.utils import shuffle
107    >>> import numpy as np
108
109    >>> diabetes = datasets.load_diabetes()
110    >>> X, y = shuffle(diabetes.data, diabetes.target, random_state=13)
111    >>> X = X.astype(np.float32)
112
113    >>> offset = int(X.shape[0] * 0.9)
114    >>> X_train, y_train = X[:offset], y[:offset]
115    >>> X_test, y_test = X[offset:], y[offset:]
116
117    >>> reg = LazyRegressor(verbose=0, ignore_warnings=False, custom_metric=None)
118    >>> models, predictions = reg.fit(X_train, X_test, y_train, y_test)
119    >>> model_dictionary = reg.provide_models(X_train, X_test, y_train, y_test)
120    >>> print(models)
121    """
122
123    def __init__(
124        self,
125        verbose=0,
126        ignore_warnings=True,
127        custom_metric=None,
128        predictions=False,
129        random_state=42,
130        regressors="all",
131        preprocess=False,
132        n_jobs=None,
133        # CustomRegressor attributes
134        obj=None,
135        n_hidden_features=5,
136        activation_name="relu",
137        a=0.01,
138        nodes_sim="sobol",
139        bias=True,
140        dropout=0,
141        direct_link=True,
142        n_clusters=2,
143        cluster_encode=True,
144        type_clust="kmeans",
145        type_scaling=("std", "std", "std"),
146        col_sample=1,
147        row_sample=1,
148        seed=123,
149        backend="cpu",
150    ):
151        self.verbose = verbose
152        self.ignore_warnings = ignore_warnings
153        self.custom_metric = custom_metric
154        self.predictions = predictions
155        self.models = {}
156        self.random_state = random_state
157        self.regressors = regressors
158        self.preprocess = preprocess
159        self.n_jobs = n_jobs
160        super().__init__(
161            obj=obj,
162            n_hidden_features=n_hidden_features,
163            activation_name=activation_name,
164            a=a,
165            nodes_sim=nodes_sim,
166            bias=bias,
167            dropout=dropout,
168            direct_link=direct_link,
169            n_clusters=n_clusters,
170            cluster_encode=cluster_encode,
171            type_clust=type_clust,
172            type_scaling=type_scaling,
173            col_sample=col_sample,
174            row_sample=row_sample,
175            seed=seed,
176            backend=backend,
177        )
178
179    def fit(self, X_train, X_test, y_train, y_test):
180        """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.
181        Parameters
182        ----------
183        X_train : array-like,
184            Training vectors, where rows is the number of samples
185            and columns is the number of features.
186        X_test : array-like,
187            Testing vectors, where rows is the number of samples
188            and columns is the number of features.
189        y_train : array-like,
190            Training vectors, where rows is the number of samples
191            and columns is the number of features.
192        y_test : array-like,
193            Testing vectors, where rows is the number of samples
194            and columns is the number of features.
195        Returns
196        -------
197        scores : Pandas DataFrame
198            Returns metrics of all the models in a Pandas DataFrame.
199        predictions : Pandas DataFrame
200            Returns predictions of all the models in a Pandas DataFrame.
201        """
202        R2 = []
203        ADJR2 = []
204        RMSE = []
205        # WIN = []
206        names = []
207        TIME = []
208        predictions = {}
209
210        if self.custom_metric:
211            CUSTOM_METRIC = []
212
213        if isinstance(X_train, np.ndarray):
214            X_train = pd.DataFrame(X_train)
215            X_test = pd.DataFrame(X_test)
216
217        numeric_features = X_train.select_dtypes(include=[np.number]).columns
218        categorical_features = X_train.select_dtypes(include=["object"]).columns
219
220        categorical_low, categorical_high = get_card_split(
221            X_train, categorical_features
222        )
223
224        if self.preprocess is True:
225            preprocessor = ColumnTransformer(
226                transformers=[
227                    ("numeric", numeric_transformer, numeric_features),
228                    (
229                        "categorical_low",
230                        categorical_transformer_low,
231                        categorical_low,
232                    ),
233                    (
234                        "categorical_high",
235                        categorical_transformer_high,
236                        categorical_high,
237                    ),
238                ]
239            )
240
241        if self.regressors == "all":
242            self.regressors = REGRESSORS
243        else:
244            try:
245                temp_list = []
246                for regressor in self.regressors:
247                    full_name = (regressor.__name__, regressor)
248                    temp_list.append(full_name)
249                self.regressors = temp_list
250            except Exception as exception:
251                print(exception)
252                print("Invalid Regressor(s)")
253
254        if self.preprocess is True:
255            for name, model in tqdm(self.regressors):  # do parallel exec
256                start = time.time()
257                try:
258                    if "random_state" in model().get_params().keys():
259                        pipe = Pipeline(
260                            steps=[
261                                ("preprocessor", preprocessor),
262                                (
263                                    "regressor",
264                                    CustomRegressor(
265                                        obj=model(
266                                            random_state=self.random_state
267                                        ),
268                                        n_hidden_features=self.n_hidden_features,
269                                        activation_name=self.activation_name,
270                                        a=self.a,
271                                        nodes_sim=self.nodes_sim,
272                                        bias=self.bias,
273                                        dropout=self.dropout,
274                                        direct_link=self.direct_link,
275                                        n_clusters=self.n_clusters,
276                                        cluster_encode=self.cluster_encode,
277                                        type_clust=self.type_clust,
278                                        type_scaling=self.type_scaling,
279                                        col_sample=self.col_sample,
280                                        row_sample=self.row_sample,
281                                        seed=self.seed,
282                                        backend=self.backend,
283                                    ),
284                                ),
285                            ]
286                        )
287                    else:
288                        pipe = Pipeline(
289                            steps=[
290                                ("preprocessor", preprocessor),
291                                (
292                                    "regressor",
293                                    CustomRegressor(
294                                        obj=model(),
295                                        n_hidden_features=self.n_hidden_features,
296                                        activation_name=self.activation_name,
297                                        a=self.a,
298                                        nodes_sim=self.nodes_sim,
299                                        bias=self.bias,
300                                        dropout=self.dropout,
301                                        direct_link=self.direct_link,
302                                        n_clusters=self.n_clusters,
303                                        cluster_encode=self.cluster_encode,
304                                        type_clust=self.type_clust,
305                                        type_scaling=self.type_scaling,
306                                        col_sample=self.col_sample,
307                                        row_sample=self.row_sample,
308                                        seed=self.seed,
309                                        backend=self.backend,
310                                    ),
311                                ),
312                            ]
313                        )
314
315                    pipe.fit(X_train, y_train)
316                    self.models[name] = pipe
317                    y_pred = pipe.predict(X_test)
318
319                    r_squared = r2_score(y_test, y_pred)
320                    adj_rsquared = adjusted_rsquared(
321                        r_squared, X_test.shape[0], X_test.shape[1]
322                    )
323                    rmse = mean_squared_error(y_test, y_pred, squared=False)
324
325                    names.append(name)
326                    R2.append(r_squared)
327                    ADJR2.append(adj_rsquared)
328                    RMSE.append(rmse)
329                    TIME.append(time.time() - start)
330
331                    if self.custom_metric:
332                        custom_metric = self.custom_metric(y_test, y_pred)
333                        CUSTOM_METRIC.append(custom_metric)
334
335                    if self.verbose > 0:
336                        scores_verbose = {
337                            "Model": name,
338                            "R-Squared": r_squared,
339                            "Adjusted R-Squared": adj_rsquared,
340                            "RMSE": rmse,
341                            "Time taken": time.time() - start,
342                        }
343
344                        if self.custom_metric:
345                            scores_verbose[self.custom_metric.__name__] = (
346                                custom_metric
347                            )
348
349                        print(scores_verbose)
350                    if self.predictions:
351                        predictions[name] = y_pred
352                except Exception as exception:
353                    if self.ignore_warnings is False:
354                        print(name + " model failed to execute")
355                        print(exception)
356
357        else:
358            for name, model in tqdm(self.regressors):  # do parallel exec
359                start = time.time()
360                try:
361                    if "random_state" in model().get_params().keys():
362                        pipe = CustomRegressor(
363                            obj=model(random_state=self.random_state),
364                            n_hidden_features=self.n_hidden_features,
365                            activation_name=self.activation_name,
366                            a=self.a,
367                            nodes_sim=self.nodes_sim,
368                            bias=self.bias,
369                            dropout=self.dropout,
370                            direct_link=self.direct_link,
371                            n_clusters=self.n_clusters,
372                            cluster_encode=self.cluster_encode,
373                            type_clust=self.type_clust,
374                            type_scaling=self.type_scaling,
375                            col_sample=self.col_sample,
376                            row_sample=self.row_sample,
377                            seed=self.seed,
378                            backend=self.backend,
379                        )
380                    else:
381                        pipe = CustomRegressor(
382                            obj=model(),
383                            n_hidden_features=self.n_hidden_features,
384                            activation_name=self.activation_name,
385                            a=self.a,
386                            nodes_sim=self.nodes_sim,
387                            bias=self.bias,
388                            dropout=self.dropout,
389                            direct_link=self.direct_link,
390                            n_clusters=self.n_clusters,
391                            cluster_encode=self.cluster_encode,
392                            type_clust=self.type_clust,
393                            type_scaling=self.type_scaling,
394                            col_sample=self.col_sample,
395                            row_sample=self.row_sample,
396                            seed=self.seed,
397                            backend=self.backend,
398                        )
399
400                    pipe.fit(X_train, y_train)
401                    self.models[name] = pipe
402                    y_pred = pipe.predict(X_test)
403
404                    r_squared = r2_score(y_test, y_pred)
405                    adj_rsquared = adjusted_rsquared(
406                        r_squared, X_test.shape[0], X_test.shape[1]
407                    )
408                    rmse = mean_squared_error(y_test, y_pred, squared=False)
409
410                    names.append(name)
411                    R2.append(r_squared)
412                    ADJR2.append(adj_rsquared)
413                    RMSE.append(rmse)
414                    TIME.append(time.time() - start)
415
416                    if self.custom_metric:
417                        custom_metric = self.custom_metric(y_test, y_pred)
418                        CUSTOM_METRIC.append(custom_metric)
419
420                    if self.verbose > 0:
421                        scores_verbose = {
422                            "Model": name,
423                            "R-Squared": r_squared,
424                            "Adjusted R-Squared": adj_rsquared,
425                            "RMSE": rmse,
426                            "Time taken": time.time() - start,
427                        }
428
429                        if self.custom_metric:
430                            scores_verbose[self.custom_metric.__name__] = (
431                                custom_metric
432                            )
433
434                        print(scores_verbose)
435                    if self.predictions:
436                        predictions[name] = y_pred
437                except Exception as exception:
438                    if self.ignore_warnings is False:
439                        print(name + " model failed to execute")
440                        print(exception)
441
442        scores = {
443            "Model": names,
444            "Adjusted R-Squared": ADJR2,
445            "R-Squared": R2,
446            "RMSE": RMSE,
447            "Time Taken": TIME,
448        }
449
450        if self.custom_metric:
451            scores[self.custom_metric.__name__] = CUSTOM_METRIC
452
453        scores = pd.DataFrame(scores)
454        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
455            "Model"
456        )
457
458        if self.predictions:
459            predictions_df = pd.DataFrame.from_dict(predictions)
460        return scores, predictions_df if self.predictions is True else scores
461
462    def provide_models(self, X_train, X_test, y_train, y_test):
463        """
464        This function returns all the model objects trained in fit function.
465        If fit is not called already, then we call fit and then return the models.
466        Parameters
467        ----------
468        X_train : array-like,
469            Training vectors, where rows is the number of samples
470            and columns is the number of features.
471        X_test : array-like,
472            Testing vectors, where rows is the number of samples
473            and columns is the number of features.
474        y_train : array-like,
475            Training vectors, where rows is the number of samples
476            and columns is the number of features.
477        y_test : array-like,
478            Testing vectors, where rows is the number of samples
479            and columns is the number of features.
480        Returns
481        -------
482        models: dict-object,
483            Returns a dictionary with each model pipeline as value
484            with key as name of models.
485        """
486        if len(self.models.keys()) == 0:
487            self.fit(X_train, X_test, y_train, y_test)
488
489        return self.models

This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's CustomRegressor

Parameters

verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. prediction : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. regressors : list, optional (default="all") When function is provided, trains the chosen regressor(s). n_jobs : int, when possible, run in parallel

Examples

>>> from lazypredict.Supervised import LazyRegressor
>>> from sklearn import datasets
>>> from sklearn.utils import shuffle
>>> import numpy as np
>>> diabetes = datasets.load_diabetes()
>>> X, y = shuffle(diabetes.data, diabetes.target, random_state=13)
>>> X = X.astype(np.float32)
>>> offset = int(X.shape[0] * 0.9)
>>> X_train, y_train = X[:offset], y[:offset]
>>> X_test, y_test = X[offset:], y[offset:]
>>> reg = LazyRegressor(verbose=0, ignore_warnings=False, custom_metric=None)
>>> models, predictions = reg.fit(X_train, X_test, y_train, y_test)
>>> model_dictionary = reg.provide_models(X_train, X_test, y_train, y_test)
>>> print(models)
LazyRegressor( verbose=0, ignore_warnings=True, custom_metric=None, predictions=False, random_state=42, regressors='all', preprocess=False, n_jobs=None, obj=None, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
123    def __init__(
124        self,
125        verbose=0,
126        ignore_warnings=True,
127        custom_metric=None,
128        predictions=False,
129        random_state=42,
130        regressors="all",
131        preprocess=False,
132        n_jobs=None,
133        # CustomRegressor attributes
134        obj=None,
135        n_hidden_features=5,
136        activation_name="relu",
137        a=0.01,
138        nodes_sim="sobol",
139        bias=True,
140        dropout=0,
141        direct_link=True,
142        n_clusters=2,
143        cluster_encode=True,
144        type_clust="kmeans",
145        type_scaling=("std", "std", "std"),
146        col_sample=1,
147        row_sample=1,
148        seed=123,
149        backend="cpu",
150    ):
151        self.verbose = verbose
152        self.ignore_warnings = ignore_warnings
153        self.custom_metric = custom_metric
154        self.predictions = predictions
155        self.models = {}
156        self.random_state = random_state
157        self.regressors = regressors
158        self.preprocess = preprocess
159        self.n_jobs = n_jobs
160        super().__init__(
161            obj=obj,
162            n_hidden_features=n_hidden_features,
163            activation_name=activation_name,
164            a=a,
165            nodes_sim=nodes_sim,
166            bias=bias,
167            dropout=dropout,
168            direct_link=direct_link,
169            n_clusters=n_clusters,
170            cluster_encode=cluster_encode,
171            type_clust=type_clust,
172            type_scaling=type_scaling,
173            col_sample=col_sample,
174            row_sample=row_sample,
175            seed=seed,
176            backend=backend,
177        )
verbose
ignore_warnings
custom_metric
predictions
models
random_state
regressors
preprocess
n_jobs
def fit(self, X_train, X_test, y_train, y_test):
179    def fit(self, X_train, X_test, y_train, y_test):
180        """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.
181        Parameters
182        ----------
183        X_train : array-like,
184            Training vectors, where rows is the number of samples
185            and columns is the number of features.
186        X_test : array-like,
187            Testing vectors, where rows is the number of samples
188            and columns is the number of features.
189        y_train : array-like,
190            Training vectors, where rows is the number of samples
191            and columns is the number of features.
192        y_test : array-like,
193            Testing vectors, where rows is the number of samples
194            and columns is the number of features.
195        Returns
196        -------
197        scores : Pandas DataFrame
198            Returns metrics of all the models in a Pandas DataFrame.
199        predictions : Pandas DataFrame
200            Returns predictions of all the models in a Pandas DataFrame.
201        """
202        R2 = []
203        ADJR2 = []
204        RMSE = []
205        # WIN = []
206        names = []
207        TIME = []
208        predictions = {}
209
210        if self.custom_metric:
211            CUSTOM_METRIC = []
212
213        if isinstance(X_train, np.ndarray):
214            X_train = pd.DataFrame(X_train)
215            X_test = pd.DataFrame(X_test)
216
217        numeric_features = X_train.select_dtypes(include=[np.number]).columns
218        categorical_features = X_train.select_dtypes(include=["object"]).columns
219
220        categorical_low, categorical_high = get_card_split(
221            X_train, categorical_features
222        )
223
224        if self.preprocess is True:
225            preprocessor = ColumnTransformer(
226                transformers=[
227                    ("numeric", numeric_transformer, numeric_features),
228                    (
229                        "categorical_low",
230                        categorical_transformer_low,
231                        categorical_low,
232                    ),
233                    (
234                        "categorical_high",
235                        categorical_transformer_high,
236                        categorical_high,
237                    ),
238                ]
239            )
240
241        if self.regressors == "all":
242            self.regressors = REGRESSORS
243        else:
244            try:
245                temp_list = []
246                for regressor in self.regressors:
247                    full_name = (regressor.__name__, regressor)
248                    temp_list.append(full_name)
249                self.regressors = temp_list
250            except Exception as exception:
251                print(exception)
252                print("Invalid Regressor(s)")
253
254        if self.preprocess is True:
255            for name, model in tqdm(self.regressors):  # do parallel exec
256                start = time.time()
257                try:
258                    if "random_state" in model().get_params().keys():
259                        pipe = Pipeline(
260                            steps=[
261                                ("preprocessor", preprocessor),
262                                (
263                                    "regressor",
264                                    CustomRegressor(
265                                        obj=model(
266                                            random_state=self.random_state
267                                        ),
268                                        n_hidden_features=self.n_hidden_features,
269                                        activation_name=self.activation_name,
270                                        a=self.a,
271                                        nodes_sim=self.nodes_sim,
272                                        bias=self.bias,
273                                        dropout=self.dropout,
274                                        direct_link=self.direct_link,
275                                        n_clusters=self.n_clusters,
276                                        cluster_encode=self.cluster_encode,
277                                        type_clust=self.type_clust,
278                                        type_scaling=self.type_scaling,
279                                        col_sample=self.col_sample,
280                                        row_sample=self.row_sample,
281                                        seed=self.seed,
282                                        backend=self.backend,
283                                    ),
284                                ),
285                            ]
286                        )
287                    else:
288                        pipe = Pipeline(
289                            steps=[
290                                ("preprocessor", preprocessor),
291                                (
292                                    "regressor",
293                                    CustomRegressor(
294                                        obj=model(),
295                                        n_hidden_features=self.n_hidden_features,
296                                        activation_name=self.activation_name,
297                                        a=self.a,
298                                        nodes_sim=self.nodes_sim,
299                                        bias=self.bias,
300                                        dropout=self.dropout,
301                                        direct_link=self.direct_link,
302                                        n_clusters=self.n_clusters,
303                                        cluster_encode=self.cluster_encode,
304                                        type_clust=self.type_clust,
305                                        type_scaling=self.type_scaling,
306                                        col_sample=self.col_sample,
307                                        row_sample=self.row_sample,
308                                        seed=self.seed,
309                                        backend=self.backend,
310                                    ),
311                                ),
312                            ]
313                        )
314
315                    pipe.fit(X_train, y_train)
316                    self.models[name] = pipe
317                    y_pred = pipe.predict(X_test)
318
319                    r_squared = r2_score(y_test, y_pred)
320                    adj_rsquared = adjusted_rsquared(
321                        r_squared, X_test.shape[0], X_test.shape[1]
322                    )
323                    rmse = mean_squared_error(y_test, y_pred, squared=False)
324
325                    names.append(name)
326                    R2.append(r_squared)
327                    ADJR2.append(adj_rsquared)
328                    RMSE.append(rmse)
329                    TIME.append(time.time() - start)
330
331                    if self.custom_metric:
332                        custom_metric = self.custom_metric(y_test, y_pred)
333                        CUSTOM_METRIC.append(custom_metric)
334
335                    if self.verbose > 0:
336                        scores_verbose = {
337                            "Model": name,
338                            "R-Squared": r_squared,
339                            "Adjusted R-Squared": adj_rsquared,
340                            "RMSE": rmse,
341                            "Time taken": time.time() - start,
342                        }
343
344                        if self.custom_metric:
345                            scores_verbose[self.custom_metric.__name__] = (
346                                custom_metric
347                            )
348
349                        print(scores_verbose)
350                    if self.predictions:
351                        predictions[name] = y_pred
352                except Exception as exception:
353                    if self.ignore_warnings is False:
354                        print(name + " model failed to execute")
355                        print(exception)
356
357        else:
358            for name, model in tqdm(self.regressors):  # do parallel exec
359                start = time.time()
360                try:
361                    if "random_state" in model().get_params().keys():
362                        pipe = CustomRegressor(
363                            obj=model(random_state=self.random_state),
364                            n_hidden_features=self.n_hidden_features,
365                            activation_name=self.activation_name,
366                            a=self.a,
367                            nodes_sim=self.nodes_sim,
368                            bias=self.bias,
369                            dropout=self.dropout,
370                            direct_link=self.direct_link,
371                            n_clusters=self.n_clusters,
372                            cluster_encode=self.cluster_encode,
373                            type_clust=self.type_clust,
374                            type_scaling=self.type_scaling,
375                            col_sample=self.col_sample,
376                            row_sample=self.row_sample,
377                            seed=self.seed,
378                            backend=self.backend,
379                        )
380                    else:
381                        pipe = CustomRegressor(
382                            obj=model(),
383                            n_hidden_features=self.n_hidden_features,
384                            activation_name=self.activation_name,
385                            a=self.a,
386                            nodes_sim=self.nodes_sim,
387                            bias=self.bias,
388                            dropout=self.dropout,
389                            direct_link=self.direct_link,
390                            n_clusters=self.n_clusters,
391                            cluster_encode=self.cluster_encode,
392                            type_clust=self.type_clust,
393                            type_scaling=self.type_scaling,
394                            col_sample=self.col_sample,
395                            row_sample=self.row_sample,
396                            seed=self.seed,
397                            backend=self.backend,
398                        )
399
400                    pipe.fit(X_train, y_train)
401                    self.models[name] = pipe
402                    y_pred = pipe.predict(X_test)
403
404                    r_squared = r2_score(y_test, y_pred)
405                    adj_rsquared = adjusted_rsquared(
406                        r_squared, X_test.shape[0], X_test.shape[1]
407                    )
408                    rmse = mean_squared_error(y_test, y_pred, squared=False)
409
410                    names.append(name)
411                    R2.append(r_squared)
412                    ADJR2.append(adj_rsquared)
413                    RMSE.append(rmse)
414                    TIME.append(time.time() - start)
415
416                    if self.custom_metric:
417                        custom_metric = self.custom_metric(y_test, y_pred)
418                        CUSTOM_METRIC.append(custom_metric)
419
420                    if self.verbose > 0:
421                        scores_verbose = {
422                            "Model": name,
423                            "R-Squared": r_squared,
424                            "Adjusted R-Squared": adj_rsquared,
425                            "RMSE": rmse,
426                            "Time taken": time.time() - start,
427                        }
428
429                        if self.custom_metric:
430                            scores_verbose[self.custom_metric.__name__] = (
431                                custom_metric
432                            )
433
434                        print(scores_verbose)
435                    if self.predictions:
436                        predictions[name] = y_pred
437                except Exception as exception:
438                    if self.ignore_warnings is False:
439                        print(name + " model failed to execute")
440                        print(exception)
441
442        scores = {
443            "Model": names,
444            "Adjusted R-Squared": ADJR2,
445            "R-Squared": R2,
446            "RMSE": RMSE,
447            "Time Taken": TIME,
448        }
449
450        if self.custom_metric:
451            scores[self.custom_metric.__name__] = CUSTOM_METRIC
452
453        scores = pd.DataFrame(scores)
454        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
455            "Model"
456        )
457
458        if self.predictions:
459            predictions_df = pd.DataFrame.from_dict(predictions)
460        return scores, predictions_df if self.predictions is True else scores

Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.

def provide_models(self, X_train, X_test, y_train, y_test):
462    def provide_models(self, X_train, X_test, y_train, y_test):
463        """
464        This function returns all the model objects trained in fit function.
465        If fit is not called already, then we call fit and then return the models.
466        Parameters
467        ----------
468        X_train : array-like,
469            Training vectors, where rows is the number of samples
470            and columns is the number of features.
471        X_test : array-like,
472            Testing vectors, where rows is the number of samples
473            and columns is the number of features.
474        y_train : array-like,
475            Training vectors, where rows is the number of samples
476            and columns is the number of features.
477        y_test : array-like,
478            Testing vectors, where rows is the number of samples
479            and columns is the number of features.
480        Returns
481        -------
482        models: dict-object,
483            Returns a dictionary with each model pipeline as value
484            with key as name of models.
485        """
486        if len(self.models.keys()) == 0:
487            self.fit(X_train, X_test, y_train, y_test)
488
489        return self.models

This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.

def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.custom.custom.Custom
obj
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
sklearn.base.RegressorMixin
score
class LazyMTS(nnetsauce.MTS):
 90class LazyMTS(MTS):
 91    """
 92    This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's MTS
 93    Parameters
 94    ----------
 95    verbose : int, optional (default=0)
 96        For the liblinear and lbfgs solvers set verbose to any positive
 97        number for verbosity.
 98    ignore_warnings : bool, optional (default=True)
 99        When set to True, the warning related to algorigms that are not able to run are ignored.
100    custom_metric : function, optional (default=None)
101        When function is provided, models are evaluated based on the custom evaluation metric provided.
102    prediction : bool, optional (default=False)
103        When set to True, the predictions of all the models models are returned as dataframe.
104    regressors : list, optional (default="all")
105        When function is provided, trains the chosen regressor(s).
106
107    Examples
108    --------
109
110    """
111
112    def __init__(
113        self,
114        verbose=0,
115        ignore_warnings=True,
116        custom_metric=None,
117        predictions=False,
118        random_state=42,
119        regressors="all",
120        preprocess=False,
121        # MTS attributes
122        obj=None,
123        n_hidden_features=5,
124        activation_name="relu",
125        a=0.01,
126        nodes_sim="sobol",
127        bias=True,
128        dropout=0,
129        direct_link=True,
130        n_clusters=2,
131        cluster_encode=True,
132        type_clust="kmeans",
133        type_scaling=("std", "std", "std"),
134        lags=1,
135        replications=None,
136        kernel=None,
137        agg="mean",
138        seed=123,
139        backend="cpu",
140        show_progress=False,
141    ):
142        self.verbose = verbose
143        self.ignore_warnings = ignore_warnings
144        self.custom_metric = custom_metric
145        self.predictions = predictions
146        self.models = {}
147        self.random_state = random_state
148        self.regressors = regressors
149        self.preprocess = preprocess
150        super().__init__(
151            obj=obj,
152            n_hidden_features=n_hidden_features,
153            activation_name=activation_name,
154            a=a,
155            nodes_sim=nodes_sim,
156            bias=bias,
157            dropout=dropout,
158            direct_link=direct_link,
159            n_clusters=n_clusters,
160            cluster_encode=cluster_encode,
161            type_clust=type_clust,
162            type_scaling=type_scaling,
163            seed=seed,
164            backend=backend,
165            lags=lags,
166            replications=replications,
167            kernel=kernel,
168            agg=agg,
169            show_progress=show_progress,
170        )
171
172    def fit(self, X_train, X_test, xreg=None, new_xreg=None, **kwargs):
173        """Fit Regression algorithms to X_train, predict and score on X_test.
174        Parameters
175        ----------
176        X_train : array-like,
177            Training vectors, where rows is the number of samples
178            and columns is the number of features.
179        X_test : array-like,
180            Testing vectors, where rows is the number of samples
181            and columns is the number of features.
182        Returns
183        -------
184        scores : Pandas DataFrame
185            Returns metrics of all the models in a Pandas DataFrame.
186        predictions : Pandas DataFrame
187            Returns predictions of all the models in a Pandas DataFrame.
188        """
189        R2 = []
190        ADJR2 = []
191        ME = []
192        MPL = []
193        RMSE = []
194        MAE = []
195        MPE = []
196        MAPE = []
197
198        # WIN = []
199        names = []
200        TIME = []
201        predictions = {}
202
203        if self.custom_metric:
204            CUSTOM_METRIC = []
205
206        if isinstance(X_train, np.ndarray):
207            X_train = pd.DataFrame(X_train)
208            X_test = pd.DataFrame(X_test)
209
210        X_train = convert_df_to_numeric(X_train)
211        X_test = convert_df_to_numeric(X_test)
212
213        numeric_features = X_train.select_dtypes(include=[np.number]).columns
214        categorical_features = X_train.select_dtypes(include=["object"]).columns
215
216        categorical_low, categorical_high = get_card_split(
217            X_train, categorical_features
218        )
219
220        if self.preprocess:
221            preprocessor = ColumnTransformer(
222                transformers=[
223                    ("numeric", numeric_transformer, numeric_features),
224                    (
225                        "categorical_low",
226                        categorical_transformer_low,
227                        categorical_low,
228                    ),
229                    (
230                        "categorical_high",
231                        categorical_transformer_high,
232                        categorical_high,
233                    ),
234                ]
235            )
236
237        if self.regressors == "all":
238            self.regressors = REGRESSORSMTS
239        else:
240            try:
241                temp_list = []
242                for regressor in self.regressors:
243                    full_name = (regressor.__name__, regressor)
244                    temp_list.append(full_name)
245                self.regressors = temp_list
246            except Exception as exception:
247                print(exception)
248                print("Invalid Regressor(s)")
249
250        if self.preprocess is True:
251            for name, model in tqdm(self.regressors):  # do parallel exec
252                start = time.time()
253                try:
254                    if "random_state" in model().get_params().keys():
255                        pipe = Pipeline(
256                            steps=[
257                                ("preprocessor", preprocessor),
258                                (
259                                    "regressor",
260                                    MTS(
261                                        obj=model(
262                                            random_state=self.random_state,
263                                            **kwargs
264                                        ),
265                                        n_hidden_features=self.n_hidden_features,
266                                        activation_name=self.activation_name,
267                                        a=self.a,
268                                        nodes_sim=self.nodes_sim,
269                                        bias=self.bias,
270                                        dropout=self.dropout,
271                                        direct_link=self.direct_link,
272                                        n_clusters=self.n_clusters,
273                                        cluster_encode=self.cluster_encode,
274                                        type_clust=self.type_clust,
275                                        type_scaling=self.type_scaling,
276                                        lags=self.lags,
277                                        replications=self.replications,
278                                        kernel=self.kernel,
279                                        agg=self.agg,
280                                        seed=self.seed,
281                                        backend=self.backend,
282                                        show_progress=self.show_progress,
283                                    ),
284                                ),
285                            ]
286                        )
287                    else:
288                        pipe = Pipeline(
289                            steps=[
290                                ("preprocessor", preprocessor),
291                                (
292                                    "regressor",
293                                    MTS(
294                                        obj=model(**kwargs),
295                                        n_hidden_features=self.n_hidden_features,
296                                        activation_name=self.activation_name,
297                                        a=self.a,
298                                        nodes_sim=self.nodes_sim,
299                                        bias=self.bias,
300                                        dropout=self.dropout,
301                                        direct_link=self.direct_link,
302                                        n_clusters=self.n_clusters,
303                                        cluster_encode=self.cluster_encode,
304                                        type_clust=self.type_clust,
305                                        type_scaling=self.type_scaling,
306                                        lags=self.lags,
307                                        replications=self.replications,
308                                        kernel=self.kernel,
309                                        agg=self.agg,
310                                        seed=self.seed,
311                                        backend=self.backend,
312                                        show_progress=self.show_progress,
313                                    ),
314                                ),
315                            ]
316                        )
317
318                    pipe.fit(X_train, **kwargs)
319                    # pipe.fit(X_train, xreg=xreg)
320
321                    self.models[name] = pipe
322                    if xreg is not None:
323                        assert (
324                            new_xreg is not None
325                        ), "xreg and new_xreg must be provided"
326                    # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg)
327                    X_pred = pipe["regressor"].predict(
328                        h=X_test.shape[0], **kwargs
329                    )
330                    rmse = mean_squared_error(X_test, X_pred, squared=False)
331                    mae = mean_absolute_error(X_test, X_pred)
332                    mpl = mean_pinball_loss(X_test, X_pred)
333
334                    names.append(name)
335                    RMSE.append(rmse)
336                    MAE.append(mae)
337                    MPL.append(mpl)
338                    TIME.append(time.time() - start)
339
340                    if self.custom_metric:
341                        custom_metric = self.custom_metric(X_test, X_pred)
342                        CUSTOM_METRIC.append(custom_metric)
343
344                    if self.verbose > 0:
345                        scores_verbose = {
346                            "Model": name,
347                            # "R-Squared": r_squared,
348                            # "Adjusted R-Squared": adj_rsquared,
349                            "RMSE": rmse,
350                            "MAE": mae,
351                            "MPL": mpl,
352                            # "MPE": mpe,
353                            # "MAPE": mape,
354                            "Time taken": time.time() - start,
355                        }
356
357                        if self.custom_metric:
358                            scores_verbose[self.custom_metric.__name__] = (
359                                custom_metric
360                            )
361
362                        print(scores_verbose)
363                    if self.predictions:
364                        predictions[name] = X_pred
365                except Exception as exception:
366                    if self.ignore_warnings is False:
367                        print(name + " model failed to execute")
368                        print(exception)
369
370        else:  # no preprocessing
371            for name, model in tqdm(self.regressors):  # do parallel exec
372                start = time.time()
373                try:
374                    if "random_state" in model().get_params().keys():
375                        pipe = MTS(
376                            obj=model(random_state=self.random_state, **kwargs),
377                            n_hidden_features=self.n_hidden_features,
378                            activation_name=self.activation_name,
379                            a=self.a,
380                            nodes_sim=self.nodes_sim,
381                            bias=self.bias,
382                            dropout=self.dropout,
383                            direct_link=self.direct_link,
384                            n_clusters=self.n_clusters,
385                            cluster_encode=self.cluster_encode,
386                            type_clust=self.type_clust,
387                            type_scaling=self.type_scaling,
388                            lags=self.lags,
389                            replications=self.replications,
390                            kernel=self.kernel,
391                            agg=self.agg,
392                            seed=self.seed,
393                            backend=self.backend,
394                            show_progress=self.show_progress,
395                        )
396                    else:
397                        pipe = MTS(
398                            obj=model(**kwargs),
399                            n_hidden_features=self.n_hidden_features,
400                            activation_name=self.activation_name,
401                            a=self.a,
402                            nodes_sim=self.nodes_sim,
403                            bias=self.bias,
404                            dropout=self.dropout,
405                            direct_link=self.direct_link,
406                            n_clusters=self.n_clusters,
407                            cluster_encode=self.cluster_encode,
408                            type_clust=self.type_clust,
409                            type_scaling=self.type_scaling,
410                            lags=self.lags,
411                            replications=self.replications,
412                            kernel=self.kernel,
413                            agg=self.agg,
414                            seed=self.seed,
415                            backend=self.backend,
416                            show_progress=self.show_progress,
417                        )
418
419                    pipe.fit(X_train, **kwargs)
420                    # pipe.fit(X_train, xreg=xreg) # DO xreg like in `ahead`
421
422                    self.models[name] = pipe
423                    if xreg is not None:
424                        assert (
425                            new_xreg is not None
426                        ), "xreg and new_xreg must be provided"
427
428                    if self.preprocess is True:
429                        X_pred = pipe["regressor"].predict(
430                            h=X_test.shape[0], **kwargs
431                        )
432                    else:
433                        X_pred = pipe.predict(
434                            h=X_test.shape[0], **kwargs
435                        )  # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) ## DO xreg like in `ahead`
436
437                    rmse = mean_squared_error(X_test, X_pred, squared=False)
438                    mae = mean_absolute_error(X_test, X_pred)
439                    mpl = mean_pinball_loss(X_test, X_pred)
440
441                    names.append(name)
442                    RMSE.append(rmse)
443                    MAE.append(mae)
444                    MPL.append(mpl)
445                    TIME.append(time.time() - start)
446
447                    if self.custom_metric:
448                        custom_metric = self.custom_metric(X_test, X_pred)
449                        CUSTOM_METRIC.append(custom_metric)
450
451                    if self.verbose > 0:
452                        scores_verbose = {
453                            "Model": name,
454                            # "R-Squared": r_squared,
455                            # "Adjusted R-Squared": adj_rsquared,
456                            "RMSE": rmse,
457                            "MAE": mae,
458                            "MPL": mpl,
459                            # "MPE": mpe,
460                            # "MAPE": mape,
461                            "Time taken": time.time() - start,
462                        }
463
464                        if self.custom_metric:
465                            scores_verbose[self.custom_metric.__name__] = (
466                                custom_metric
467                            )
468
469                        print(scores_verbose)
470                    if self.predictions:
471                        predictions[name] = X_pred
472                except Exception as exception:
473                    if self.ignore_warnings is False:
474                        print(name + " model failed to execute")
475                        print(exception)
476
477        scores = {
478            "Model": names,
479            # "Adjusted R-Squared": ADJR2,
480            # "R-Squared": R2,
481            "RMSE": RMSE,
482            "MAE": MAE,
483            "MPL": MPL,
484            # "MPE": MPE,
485            # "MAPE": MAPE,
486            "Time Taken": TIME,
487        }
488
489        if self.custom_metric:
490            scores[self.custom_metric.__name__] = CUSTOM_METRIC
491
492        scores = pd.DataFrame(scores)
493        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
494            "Model"
495        )
496
497        if self.predictions:
498            predictions_df = pd.DataFrame.from_dict(predictions)
499        return scores, predictions_df if self.predictions is True else scores
500
501    def provide_models(self, X_train, X_test):
502        """
503        This function returns all the model objects trained in fit function.
504        If fit is not called already, then we call fit and then return the models.
505        Parameters
506        ----------
507        X_train : array-like,
508            Training vectors, where rows is the number of samples
509            and columns is the number of features.
510        X_test : array-like,
511            Testing vectors, where rows is the number of samples
512            and columns is the number of features.
513        Returns
514        -------
515        models: dict-object,
516            Returns a dictionary with each model pipeline as value
517            with key as name of models.
518        """
519        if len(self.models.keys()) == 0:
520            self.fit(X_train, X_test)
521
522        return self.models

This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's MTS

Parameters

verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. prediction : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. regressors : list, optional (default="all") When function is provided, trains the chosen regressor(s).

Examples

LazyMTS( verbose=0, ignore_warnings=True, custom_metric=None, predictions=False, random_state=42, regressors='all', preprocess=False, obj=None, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), lags=1, replications=None, kernel=None, agg='mean', seed=123, backend='cpu', show_progress=False)
112    def __init__(
113        self,
114        verbose=0,
115        ignore_warnings=True,
116        custom_metric=None,
117        predictions=False,
118        random_state=42,
119        regressors="all",
120        preprocess=False,
121        # MTS attributes
122        obj=None,
123        n_hidden_features=5,
124        activation_name="relu",
125        a=0.01,
126        nodes_sim="sobol",
127        bias=True,
128        dropout=0,
129        direct_link=True,
130        n_clusters=2,
131        cluster_encode=True,
132        type_clust="kmeans",
133        type_scaling=("std", "std", "std"),
134        lags=1,
135        replications=None,
136        kernel=None,
137        agg="mean",
138        seed=123,
139        backend="cpu",
140        show_progress=False,
141    ):
142        self.verbose = verbose
143        self.ignore_warnings = ignore_warnings
144        self.custom_metric = custom_metric
145        self.predictions = predictions
146        self.models = {}
147        self.random_state = random_state
148        self.regressors = regressors
149        self.preprocess = preprocess
150        super().__init__(
151            obj=obj,
152            n_hidden_features=n_hidden_features,
153            activation_name=activation_name,
154            a=a,
155            nodes_sim=nodes_sim,
156            bias=bias,
157            dropout=dropout,
158            direct_link=direct_link,
159            n_clusters=n_clusters,
160            cluster_encode=cluster_encode,
161            type_clust=type_clust,
162            type_scaling=type_scaling,
163            seed=seed,
164            backend=backend,
165            lags=lags,
166            replications=replications,
167            kernel=kernel,
168            agg=agg,
169            show_progress=show_progress,
170        )
verbose
ignore_warnings
custom_metric
predictions
models
random_state
regressors
preprocess
def fit(self, X_train, X_test, xreg=None, new_xreg=None, **kwargs):
172    def fit(self, X_train, X_test, xreg=None, new_xreg=None, **kwargs):
173        """Fit Regression algorithms to X_train, predict and score on X_test.
174        Parameters
175        ----------
176        X_train : array-like,
177            Training vectors, where rows is the number of samples
178            and columns is the number of features.
179        X_test : array-like,
180            Testing vectors, where rows is the number of samples
181            and columns is the number of features.
182        Returns
183        -------
184        scores : Pandas DataFrame
185            Returns metrics of all the models in a Pandas DataFrame.
186        predictions : Pandas DataFrame
187            Returns predictions of all the models in a Pandas DataFrame.
188        """
189        R2 = []
190        ADJR2 = []
191        ME = []
192        MPL = []
193        RMSE = []
194        MAE = []
195        MPE = []
196        MAPE = []
197
198        # WIN = []
199        names = []
200        TIME = []
201        predictions = {}
202
203        if self.custom_metric:
204            CUSTOM_METRIC = []
205
206        if isinstance(X_train, np.ndarray):
207            X_train = pd.DataFrame(X_train)
208            X_test = pd.DataFrame(X_test)
209
210        X_train = convert_df_to_numeric(X_train)
211        X_test = convert_df_to_numeric(X_test)
212
213        numeric_features = X_train.select_dtypes(include=[np.number]).columns
214        categorical_features = X_train.select_dtypes(include=["object"]).columns
215
216        categorical_low, categorical_high = get_card_split(
217            X_train, categorical_features
218        )
219
220        if self.preprocess:
221            preprocessor = ColumnTransformer(
222                transformers=[
223                    ("numeric", numeric_transformer, numeric_features),
224                    (
225                        "categorical_low",
226                        categorical_transformer_low,
227                        categorical_low,
228                    ),
229                    (
230                        "categorical_high",
231                        categorical_transformer_high,
232                        categorical_high,
233                    ),
234                ]
235            )
236
237        if self.regressors == "all":
238            self.regressors = REGRESSORSMTS
239        else:
240            try:
241                temp_list = []
242                for regressor in self.regressors:
243                    full_name = (regressor.__name__, regressor)
244                    temp_list.append(full_name)
245                self.regressors = temp_list
246            except Exception as exception:
247                print(exception)
248                print("Invalid Regressor(s)")
249
250        if self.preprocess is True:
251            for name, model in tqdm(self.regressors):  # do parallel exec
252                start = time.time()
253                try:
254                    if "random_state" in model().get_params().keys():
255                        pipe = Pipeline(
256                            steps=[
257                                ("preprocessor", preprocessor),
258                                (
259                                    "regressor",
260                                    MTS(
261                                        obj=model(
262                                            random_state=self.random_state,
263                                            **kwargs
264                                        ),
265                                        n_hidden_features=self.n_hidden_features,
266                                        activation_name=self.activation_name,
267                                        a=self.a,
268                                        nodes_sim=self.nodes_sim,
269                                        bias=self.bias,
270                                        dropout=self.dropout,
271                                        direct_link=self.direct_link,
272                                        n_clusters=self.n_clusters,
273                                        cluster_encode=self.cluster_encode,
274                                        type_clust=self.type_clust,
275                                        type_scaling=self.type_scaling,
276                                        lags=self.lags,
277                                        replications=self.replications,
278                                        kernel=self.kernel,
279                                        agg=self.agg,
280                                        seed=self.seed,
281                                        backend=self.backend,
282                                        show_progress=self.show_progress,
283                                    ),
284                                ),
285                            ]
286                        )
287                    else:
288                        pipe = Pipeline(
289                            steps=[
290                                ("preprocessor", preprocessor),
291                                (
292                                    "regressor",
293                                    MTS(
294                                        obj=model(**kwargs),
295                                        n_hidden_features=self.n_hidden_features,
296                                        activation_name=self.activation_name,
297                                        a=self.a,
298                                        nodes_sim=self.nodes_sim,
299                                        bias=self.bias,
300                                        dropout=self.dropout,
301                                        direct_link=self.direct_link,
302                                        n_clusters=self.n_clusters,
303                                        cluster_encode=self.cluster_encode,
304                                        type_clust=self.type_clust,
305                                        type_scaling=self.type_scaling,
306                                        lags=self.lags,
307                                        replications=self.replications,
308                                        kernel=self.kernel,
309                                        agg=self.agg,
310                                        seed=self.seed,
311                                        backend=self.backend,
312                                        show_progress=self.show_progress,
313                                    ),
314                                ),
315                            ]
316                        )
317
318                    pipe.fit(X_train, **kwargs)
319                    # pipe.fit(X_train, xreg=xreg)
320
321                    self.models[name] = pipe
322                    if xreg is not None:
323                        assert (
324                            new_xreg is not None
325                        ), "xreg and new_xreg must be provided"
326                    # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg)
327                    X_pred = pipe["regressor"].predict(
328                        h=X_test.shape[0], **kwargs
329                    )
330                    rmse = mean_squared_error(X_test, X_pred, squared=False)
331                    mae = mean_absolute_error(X_test, X_pred)
332                    mpl = mean_pinball_loss(X_test, X_pred)
333
334                    names.append(name)
335                    RMSE.append(rmse)
336                    MAE.append(mae)
337                    MPL.append(mpl)
338                    TIME.append(time.time() - start)
339
340                    if self.custom_metric:
341                        custom_metric = self.custom_metric(X_test, X_pred)
342                        CUSTOM_METRIC.append(custom_metric)
343
344                    if self.verbose > 0:
345                        scores_verbose = {
346                            "Model": name,
347                            # "R-Squared": r_squared,
348                            # "Adjusted R-Squared": adj_rsquared,
349                            "RMSE": rmse,
350                            "MAE": mae,
351                            "MPL": mpl,
352                            # "MPE": mpe,
353                            # "MAPE": mape,
354                            "Time taken": time.time() - start,
355                        }
356
357                        if self.custom_metric:
358                            scores_verbose[self.custom_metric.__name__] = (
359                                custom_metric
360                            )
361
362                        print(scores_verbose)
363                    if self.predictions:
364                        predictions[name] = X_pred
365                except Exception as exception:
366                    if self.ignore_warnings is False:
367                        print(name + " model failed to execute")
368                        print(exception)
369
370        else:  # no preprocessing
371            for name, model in tqdm(self.regressors):  # do parallel exec
372                start = time.time()
373                try:
374                    if "random_state" in model().get_params().keys():
375                        pipe = MTS(
376                            obj=model(random_state=self.random_state, **kwargs),
377                            n_hidden_features=self.n_hidden_features,
378                            activation_name=self.activation_name,
379                            a=self.a,
380                            nodes_sim=self.nodes_sim,
381                            bias=self.bias,
382                            dropout=self.dropout,
383                            direct_link=self.direct_link,
384                            n_clusters=self.n_clusters,
385                            cluster_encode=self.cluster_encode,
386                            type_clust=self.type_clust,
387                            type_scaling=self.type_scaling,
388                            lags=self.lags,
389                            replications=self.replications,
390                            kernel=self.kernel,
391                            agg=self.agg,
392                            seed=self.seed,
393                            backend=self.backend,
394                            show_progress=self.show_progress,
395                        )
396                    else:
397                        pipe = MTS(
398                            obj=model(**kwargs),
399                            n_hidden_features=self.n_hidden_features,
400                            activation_name=self.activation_name,
401                            a=self.a,
402                            nodes_sim=self.nodes_sim,
403                            bias=self.bias,
404                            dropout=self.dropout,
405                            direct_link=self.direct_link,
406                            n_clusters=self.n_clusters,
407                            cluster_encode=self.cluster_encode,
408                            type_clust=self.type_clust,
409                            type_scaling=self.type_scaling,
410                            lags=self.lags,
411                            replications=self.replications,
412                            kernel=self.kernel,
413                            agg=self.agg,
414                            seed=self.seed,
415                            backend=self.backend,
416                            show_progress=self.show_progress,
417                        )
418
419                    pipe.fit(X_train, **kwargs)
420                    # pipe.fit(X_train, xreg=xreg) # DO xreg like in `ahead`
421
422                    self.models[name] = pipe
423                    if xreg is not None:
424                        assert (
425                            new_xreg is not None
426                        ), "xreg and new_xreg must be provided"
427
428                    if self.preprocess is True:
429                        X_pred = pipe["regressor"].predict(
430                            h=X_test.shape[0], **kwargs
431                        )
432                    else:
433                        X_pred = pipe.predict(
434                            h=X_test.shape[0], **kwargs
435                        )  # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) ## DO xreg like in `ahead`
436
437                    rmse = mean_squared_error(X_test, X_pred, squared=False)
438                    mae = mean_absolute_error(X_test, X_pred)
439                    mpl = mean_pinball_loss(X_test, X_pred)
440
441                    names.append(name)
442                    RMSE.append(rmse)
443                    MAE.append(mae)
444                    MPL.append(mpl)
445                    TIME.append(time.time() - start)
446
447                    if self.custom_metric:
448                        custom_metric = self.custom_metric(X_test, X_pred)
449                        CUSTOM_METRIC.append(custom_metric)
450
451                    if self.verbose > 0:
452                        scores_verbose = {
453                            "Model": name,
454                            # "R-Squared": r_squared,
455                            # "Adjusted R-Squared": adj_rsquared,
456                            "RMSE": rmse,
457                            "MAE": mae,
458                            "MPL": mpl,
459                            # "MPE": mpe,
460                            # "MAPE": mape,
461                            "Time taken": time.time() - start,
462                        }
463
464                        if self.custom_metric:
465                            scores_verbose[self.custom_metric.__name__] = (
466                                custom_metric
467                            )
468
469                        print(scores_verbose)
470                    if self.predictions:
471                        predictions[name] = X_pred
472                except Exception as exception:
473                    if self.ignore_warnings is False:
474                        print(name + " model failed to execute")
475                        print(exception)
476
477        scores = {
478            "Model": names,
479            # "Adjusted R-Squared": ADJR2,
480            # "R-Squared": R2,
481            "RMSE": RMSE,
482            "MAE": MAE,
483            "MPL": MPL,
484            # "MPE": MPE,
485            # "MAPE": MAPE,
486            "Time Taken": TIME,
487        }
488
489        if self.custom_metric:
490            scores[self.custom_metric.__name__] = CUSTOM_METRIC
491
492        scores = pd.DataFrame(scores)
493        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
494            "Model"
495        )
496
497        if self.predictions:
498            predictions_df = pd.DataFrame.from_dict(predictions)
499        return scores, predictions_df if self.predictions is True else scores

Fit Regression algorithms to X_train, predict and score on X_test.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.

def provide_models(self, X_train, X_test):
501    def provide_models(self, X_train, X_test):
502        """
503        This function returns all the model objects trained in fit function.
504        If fit is not called already, then we call fit and then return the models.
505        Parameters
506        ----------
507        X_train : array-like,
508            Training vectors, where rows is the number of samples
509            and columns is the number of features.
510        X_test : array-like,
511            Testing vectors, where rows is the number of samples
512            and columns is the number of features.
513        Returns
514        -------
515        models: dict-object,
516            Returns a dictionary with each model pipeline as value
517            with key as name of models.
518        """
519        if len(self.models.keys()) == 0:
520            self.fit(X_train, X_test)
521
522        return self.models

This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.

def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_predict_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class LazyDeepClassifier(nnetsauce.custom.custom.Custom, sklearn.base.ClassifierMixin):
 81class LazyDeepClassifier(Custom, ClassifierMixin):
 82    """
 83    This module helps in fitting to all the classification algorithms that are available in Scikit-learn to nnetsauce's CustomClassifier
 84    Parameters
 85    ----------
 86    verbose : int, optional (default=0)
 87        For the liblinear and lbfgs solvers set verbose to any positive
 88        number for verbosity.
 89    ignore_warnings : bool, optional (default=True)
 90        When set to True, the warning related to algorigms that are not able to run are ignored.
 91    custom_metric : function, optional (default=None)
 92        When function is provided, models are evaluated based on the custom evaluation metric provided.
 93    prediction : bool, optional (default=False)
 94        When set to True, the predictions of all the models models are returned as dataframe.
 95    classifiers : list, optional (default="all")
 96        When function is provided, trains the chosen classifier(s).
 97    n_jobs : int, when possible, run in parallel
 98
 99    Examples
100    --------
101    >>> import nnetsauce as ns
102    >>> from sklearn.datasets import load_breast_cancer
103    >>> from sklearn.model_selection import train_test_split
104    >>> data = load_breast_cancer()
105    >>> X = data.data
106    >>> y= data.target
107    >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
108    >>> clf = ns.LazyDeepClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
109    >>> models, predictions = clf.fit(X_train, X_test, y_train, y_test)
110    >>> model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
111    >>> print(models)
112    """
113
114    def __init__(
115        self,
116        verbose=0,
117        ignore_warnings=True,
118        custom_metric=None,
119        predictions=False,
120        random_state=42,
121        classifiers="all",
122        n_jobs=None,
123        # Defining depth
124        n_layers=3,
125        # CustomClassifier attributes
126        obj=None,
127        n_hidden_features=5,
128        activation_name="relu",
129        a=0.01,
130        nodes_sim="sobol",
131        bias=True,
132        dropout=0,
133        direct_link=True,
134        n_clusters=2,
135        cluster_encode=True,
136        type_clust="kmeans",
137        type_scaling=("std", "std", "std"),
138        col_sample=1,
139        row_sample=1,
140        seed=123,
141        backend="cpu",
142    ):
143        self.verbose = verbose
144        self.ignore_warnings = ignore_warnings
145        self.custom_metric = custom_metric
146        self.predictions = predictions
147        self.models = {}
148        self.random_state = random_state
149        self.classifiers = classifiers
150        self.n_layers = n_layers - 1
151        self.n_jobs = n_jobs
152        super().__init__(
153            obj=obj,
154            n_hidden_features=n_hidden_features,
155            activation_name=activation_name,
156            a=a,
157            nodes_sim=nodes_sim,
158            bias=bias,
159            dropout=dropout,
160            direct_link=direct_link,
161            n_clusters=n_clusters,
162            cluster_encode=cluster_encode,
163            type_clust=type_clust,
164            type_scaling=type_scaling,
165            col_sample=col_sample,
166            row_sample=row_sample,
167            seed=seed,
168            backend=backend,
169        )
170
171    def fit(self, X_train, X_test, y_train, y_test):
172        """Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test.
173        Parameters
174        ----------
175        X_train : array-like,
176            Training vectors, where rows is the number of samples
177            and columns is the number of features.
178        X_test : array-like,
179            Testing vectors, where rows is the number of samples
180            and columns is the number of features.
181        y_train : array-like,
182            Training vectors, where rows is the number of samples
183            and columns is the number of features.
184        y_test : array-like,
185            Testing vectors, where rows is the number of samples
186            and columns is the number of features.
187        Returns
188        -------
189        scores : Pandas DataFrame
190            Returns metrics of all the models in a Pandas DataFrame.
191        predictions : Pandas DataFrame
192            Returns predictions of all the models in a Pandas DataFrame.
193        """
194        Accuracy = []
195        B_Accuracy = []
196        ROC_AUC = []
197        F1 = []
198        names = []
199        TIME = []
200        predictions = {}
201
202        if self.custom_metric is not None:
203            CUSTOM_METRIC = []
204
205        if isinstance(X_train, np.ndarray):
206            X_train = pd.DataFrame(X_train)
207            X_test = pd.DataFrame(X_test)
208
209        numeric_features = X_train.select_dtypes(include=[np.number]).columns
210        categorical_features = X_train.select_dtypes(include=["object"]).columns
211
212        categorical_low, categorical_high = get_card_split(
213            X_train, categorical_features
214        )
215
216        if self.classifiers == "all":
217            self.classifiers = [
218                item
219                for sublist in [
220                    CLASSIFIERS,
221                    MULTITASKCLASSIFIERS,
222                    SIMPLEMULTITASKCLASSIFIERS,
223                ]
224                for item in sublist
225            ]
226        else:
227            try:
228                temp_list = []
229                for classifier in self.classifiers:
230                    full_name = (classifier.__name__, classifier)
231                    temp_list.append(full_name)
232                self.classifiers = temp_list
233            except Exception as exception:
234                print(exception)
235                print("Invalid Classifier(s)")
236
237        for name, model in tqdm(self.classifiers):  # do parallel exec
238            start = time.time()
239            try:
240                if "random_state" in model().get_params().keys():
241                    layer_clf = CustomClassifier(
242                        obj=model(random_state=self.random_state),
243                        n_hidden_features=self.n_hidden_features,
244                        activation_name=self.activation_name,
245                        a=self.a,
246                        nodes_sim=self.nodes_sim,
247                        bias=self.bias,
248                        dropout=self.dropout,
249                        direct_link=self.direct_link,
250                        n_clusters=self.n_clusters,
251                        cluster_encode=self.cluster_encode,
252                        type_clust=self.type_clust,
253                        type_scaling=self.type_scaling,
254                        col_sample=self.col_sample,
255                        row_sample=self.row_sample,
256                        seed=self.seed,
257                        backend=self.backend,
258                    )
259
260                else:
261                    layer_clf = CustomClassifier(
262                        obj=model(),
263                        n_hidden_features=self.n_hidden_features,
264                        activation_name=self.activation_name,
265                        a=self.a,
266                        nodes_sim=self.nodes_sim,
267                        bias=self.bias,
268                        dropout=self.dropout,
269                        direct_link=self.direct_link,
270                        n_clusters=self.n_clusters,
271                        cluster_encode=self.cluster_encode,
272                        type_clust=self.type_clust,
273                        type_scaling=self.type_scaling,
274                        col_sample=self.col_sample,
275                        row_sample=self.row_sample,
276                        seed=self.seed,
277                        backend=self.backend,
278                    )
279
280                layer_clf.fit(X_train, y_train)
281
282                for _ in range(self.n_layers):
283                    layer_clf = deepcopy(
284                        CustomClassifier(
285                            obj=layer_clf,
286                            n_hidden_features=self.n_hidden_features,
287                            activation_name=self.activation_name,
288                            a=self.a,
289                            nodes_sim=self.nodes_sim,
290                            bias=self.bias,
291                            dropout=self.dropout,
292                            direct_link=self.direct_link,
293                            n_clusters=self.n_clusters,
294                            cluster_encode=self.cluster_encode,
295                            type_clust=self.type_clust,
296                            type_scaling=self.type_scaling,
297                            col_sample=self.col_sample,
298                            row_sample=self.row_sample,
299                            seed=self.seed,
300                            backend=self.backend,
301                        )
302                    )
303
304                    # layer_clf.fit(X_train, y_train)
305
306                layer_clf.fit(X_train, y_train)
307
308                self.models[name] = layer_clf
309                y_pred = layer_clf.predict(X_test)
310                accuracy = accuracy_score(y_test, y_pred, normalize=True)
311                b_accuracy = balanced_accuracy_score(y_test, y_pred)
312                f1 = f1_score(y_test, y_pred, average="weighted")
313                try:
314                    roc_auc = roc_auc_score(y_test, y_pred)
315                except Exception as exception:
316                    roc_auc = None
317                    if self.ignore_warnings is False:
318                        print("ROC AUC couldn't be calculated for " + name)
319                        print(exception)
320                names.append(name)
321                Accuracy.append(accuracy)
322                B_Accuracy.append(b_accuracy)
323                ROC_AUC.append(roc_auc)
324                F1.append(f1)
325                TIME.append(time.time() - start)
326                if self.custom_metric is not None:
327                    custom_metric = self.custom_metric(y_test, y_pred)
328                    CUSTOM_METRIC.append(custom_metric)
329                if self.verbose > 0:
330                    if self.custom_metric is not None:
331                        print(
332                            {
333                                "Model": name,
334                                "Accuracy": accuracy,
335                                "Balanced Accuracy": b_accuracy,
336                                "ROC AUC": roc_auc,
337                                "F1 Score": f1,
338                                self.custom_metric.__name__: custom_metric,
339                                "Time taken": time.time() - start,
340                            }
341                        )
342                    else:
343                        print(
344                            {
345                                "Model": name,
346                                "Accuracy": accuracy,
347                                "Balanced Accuracy": b_accuracy,
348                                "ROC AUC": roc_auc,
349                                "F1 Score": f1,
350                                "Time taken": time.time() - start,
351                            }
352                        )
353                if self.predictions:
354                    predictions[name] = y_pred
355            except Exception as exception:
356                if self.ignore_warnings is False:
357                    print(name + " model failed to execute")
358                    print(exception)
359
360        if self.custom_metric is None:
361            scores = pd.DataFrame(
362                {
363                    "Model": names,
364                    "Accuracy": Accuracy,
365                    "Balanced Accuracy": B_Accuracy,
366                    "ROC AUC": ROC_AUC,
367                    "F1 Score": F1,
368                    "Time Taken": TIME,
369                }
370            )
371        else:
372            scores = pd.DataFrame(
373                {
374                    "Model": names,
375                    "Accuracy": Accuracy,
376                    "Balanced Accuracy": B_Accuracy,
377                    "ROC AUC": ROC_AUC,
378                    "F1 Score": F1,
379                    self.custom_metric.__name__: CUSTOM_METRIC,
380                    "Time Taken": TIME,
381                }
382            )
383        scores = scores.sort_values(by="Accuracy", ascending=False).set_index(
384            "Model"
385        )
386
387        if self.predictions:
388            predictions_df = pd.DataFrame.from_dict(predictions)
389        return scores, predictions_df if self.predictions is True else scores
390
391    def provide_models(self, X_train, X_test, y_train, y_test):
392        """
393        This function returns all the model objects trained in fit function.
394        If fit is not called already, then we call fit and then return the models.
395        Parameters
396        ----------
397        X_train : array-like,
398            Training vectors, where rows is the number of samples
399            and columns is the number of features.
400        X_test : array-like,
401            Testing vectors, where rows is the number of samples
402            and columns is the number of features.
403        y_train : array-like,
404            Training vectors, where rows is the number of samples
405            and columns is the number of features.
406        y_test : array-like,
407            Testing vectors, where rows is the number of samples
408            and columns is the number of features.
409        Returns
410        -------
411        models: dict-object,
412            Returns a dictionary with each model pipeline as value
413            with key as name of models.
414        """
415        if len(self.models.keys()) == 0:
416            self.fit(X_train, X_test, y_train, y_test)
417
418        return self.models

This module helps in fitting to all the classification algorithms that are available in Scikit-learn to nnetsauce's CustomClassifier

Parameters

verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. prediction : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. classifiers : list, optional (default="all") When function is provided, trains the chosen classifier(s). n_jobs : int, when possible, run in parallel

Examples

>>> import nnetsauce as ns
>>> from sklearn.datasets import load_breast_cancer
>>> from sklearn.model_selection import train_test_split
>>> data = load_breast_cancer()
>>> X = data.data
>>> y= data.target
>>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
>>> clf = ns.LazyDeepClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
>>> models, predictions = clf.fit(X_train, X_test, y_train, y_test)
>>> model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
>>> print(models)
LazyDeepClassifier( verbose=0, ignore_warnings=True, custom_metric=None, predictions=False, random_state=42, classifiers='all', n_jobs=None, n_layers=3, obj=None, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
114    def __init__(
115        self,
116        verbose=0,
117        ignore_warnings=True,
118        custom_metric=None,
119        predictions=False,
120        random_state=42,
121        classifiers="all",
122        n_jobs=None,
123        # Defining depth
124        n_layers=3,
125        # CustomClassifier attributes
126        obj=None,
127        n_hidden_features=5,
128        activation_name="relu",
129        a=0.01,
130        nodes_sim="sobol",
131        bias=True,
132        dropout=0,
133        direct_link=True,
134        n_clusters=2,
135        cluster_encode=True,
136        type_clust="kmeans",
137        type_scaling=("std", "std", "std"),
138        col_sample=1,
139        row_sample=1,
140        seed=123,
141        backend="cpu",
142    ):
143        self.verbose = verbose
144        self.ignore_warnings = ignore_warnings
145        self.custom_metric = custom_metric
146        self.predictions = predictions
147        self.models = {}
148        self.random_state = random_state
149        self.classifiers = classifiers
150        self.n_layers = n_layers - 1
151        self.n_jobs = n_jobs
152        super().__init__(
153            obj=obj,
154            n_hidden_features=n_hidden_features,
155            activation_name=activation_name,
156            a=a,
157            nodes_sim=nodes_sim,
158            bias=bias,
159            dropout=dropout,
160            direct_link=direct_link,
161            n_clusters=n_clusters,
162            cluster_encode=cluster_encode,
163            type_clust=type_clust,
164            type_scaling=type_scaling,
165            col_sample=col_sample,
166            row_sample=row_sample,
167            seed=seed,
168            backend=backend,
169        )
verbose
ignore_warnings
custom_metric
predictions
models
random_state
classifiers
n_layers
n_jobs
def fit(self, X_train, X_test, y_train, y_test):
171    def fit(self, X_train, X_test, y_train, y_test):
172        """Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test.
173        Parameters
174        ----------
175        X_train : array-like,
176            Training vectors, where rows is the number of samples
177            and columns is the number of features.
178        X_test : array-like,
179            Testing vectors, where rows is the number of samples
180            and columns is the number of features.
181        y_train : array-like,
182            Training vectors, where rows is the number of samples
183            and columns is the number of features.
184        y_test : array-like,
185            Testing vectors, where rows is the number of samples
186            and columns is the number of features.
187        Returns
188        -------
189        scores : Pandas DataFrame
190            Returns metrics of all the models in a Pandas DataFrame.
191        predictions : Pandas DataFrame
192            Returns predictions of all the models in a Pandas DataFrame.
193        """
194        Accuracy = []
195        B_Accuracy = []
196        ROC_AUC = []
197        F1 = []
198        names = []
199        TIME = []
200        predictions = {}
201
202        if self.custom_metric is not None:
203            CUSTOM_METRIC = []
204
205        if isinstance(X_train, np.ndarray):
206            X_train = pd.DataFrame(X_train)
207            X_test = pd.DataFrame(X_test)
208
209        numeric_features = X_train.select_dtypes(include=[np.number]).columns
210        categorical_features = X_train.select_dtypes(include=["object"]).columns
211
212        categorical_low, categorical_high = get_card_split(
213            X_train, categorical_features
214        )
215
216        if self.classifiers == "all":
217            self.classifiers = [
218                item
219                for sublist in [
220                    CLASSIFIERS,
221                    MULTITASKCLASSIFIERS,
222                    SIMPLEMULTITASKCLASSIFIERS,
223                ]
224                for item in sublist
225            ]
226        else:
227            try:
228                temp_list = []
229                for classifier in self.classifiers:
230                    full_name = (classifier.__name__, classifier)
231                    temp_list.append(full_name)
232                self.classifiers = temp_list
233            except Exception as exception:
234                print(exception)
235                print("Invalid Classifier(s)")
236
237        for name, model in tqdm(self.classifiers):  # do parallel exec
238            start = time.time()
239            try:
240                if "random_state" in model().get_params().keys():
241                    layer_clf = CustomClassifier(
242                        obj=model(random_state=self.random_state),
243                        n_hidden_features=self.n_hidden_features,
244                        activation_name=self.activation_name,
245                        a=self.a,
246                        nodes_sim=self.nodes_sim,
247                        bias=self.bias,
248                        dropout=self.dropout,
249                        direct_link=self.direct_link,
250                        n_clusters=self.n_clusters,
251                        cluster_encode=self.cluster_encode,
252                        type_clust=self.type_clust,
253                        type_scaling=self.type_scaling,
254                        col_sample=self.col_sample,
255                        row_sample=self.row_sample,
256                        seed=self.seed,
257                        backend=self.backend,
258                    )
259
260                else:
261                    layer_clf = CustomClassifier(
262                        obj=model(),
263                        n_hidden_features=self.n_hidden_features,
264                        activation_name=self.activation_name,
265                        a=self.a,
266                        nodes_sim=self.nodes_sim,
267                        bias=self.bias,
268                        dropout=self.dropout,
269                        direct_link=self.direct_link,
270                        n_clusters=self.n_clusters,
271                        cluster_encode=self.cluster_encode,
272                        type_clust=self.type_clust,
273                        type_scaling=self.type_scaling,
274                        col_sample=self.col_sample,
275                        row_sample=self.row_sample,
276                        seed=self.seed,
277                        backend=self.backend,
278                    )
279
280                layer_clf.fit(X_train, y_train)
281
282                for _ in range(self.n_layers):
283                    layer_clf = deepcopy(
284                        CustomClassifier(
285                            obj=layer_clf,
286                            n_hidden_features=self.n_hidden_features,
287                            activation_name=self.activation_name,
288                            a=self.a,
289                            nodes_sim=self.nodes_sim,
290                            bias=self.bias,
291                            dropout=self.dropout,
292                            direct_link=self.direct_link,
293                            n_clusters=self.n_clusters,
294                            cluster_encode=self.cluster_encode,
295                            type_clust=self.type_clust,
296                            type_scaling=self.type_scaling,
297                            col_sample=self.col_sample,
298                            row_sample=self.row_sample,
299                            seed=self.seed,
300                            backend=self.backend,
301                        )
302                    )
303
304                    # layer_clf.fit(X_train, y_train)
305
306                layer_clf.fit(X_train, y_train)
307
308                self.models[name] = layer_clf
309                y_pred = layer_clf.predict(X_test)
310                accuracy = accuracy_score(y_test, y_pred, normalize=True)
311                b_accuracy = balanced_accuracy_score(y_test, y_pred)
312                f1 = f1_score(y_test, y_pred, average="weighted")
313                try:
314                    roc_auc = roc_auc_score(y_test, y_pred)
315                except Exception as exception:
316                    roc_auc = None
317                    if self.ignore_warnings is False:
318                        print("ROC AUC couldn't be calculated for " + name)
319                        print(exception)
320                names.append(name)
321                Accuracy.append(accuracy)
322                B_Accuracy.append(b_accuracy)
323                ROC_AUC.append(roc_auc)
324                F1.append(f1)
325                TIME.append(time.time() - start)
326                if self.custom_metric is not None:
327                    custom_metric = self.custom_metric(y_test, y_pred)
328                    CUSTOM_METRIC.append(custom_metric)
329                if self.verbose > 0:
330                    if self.custom_metric is not None:
331                        print(
332                            {
333                                "Model": name,
334                                "Accuracy": accuracy,
335                                "Balanced Accuracy": b_accuracy,
336                                "ROC AUC": roc_auc,
337                                "F1 Score": f1,
338                                self.custom_metric.__name__: custom_metric,
339                                "Time taken": time.time() - start,
340                            }
341                        )
342                    else:
343                        print(
344                            {
345                                "Model": name,
346                                "Accuracy": accuracy,
347                                "Balanced Accuracy": b_accuracy,
348                                "ROC AUC": roc_auc,
349                                "F1 Score": f1,
350                                "Time taken": time.time() - start,
351                            }
352                        )
353                if self.predictions:
354                    predictions[name] = y_pred
355            except Exception as exception:
356                if self.ignore_warnings is False:
357                    print(name + " model failed to execute")
358                    print(exception)
359
360        if self.custom_metric is None:
361            scores = pd.DataFrame(
362                {
363                    "Model": names,
364                    "Accuracy": Accuracy,
365                    "Balanced Accuracy": B_Accuracy,
366                    "ROC AUC": ROC_AUC,
367                    "F1 Score": F1,
368                    "Time Taken": TIME,
369                }
370            )
371        else:
372            scores = pd.DataFrame(
373                {
374                    "Model": names,
375                    "Accuracy": Accuracy,
376                    "Balanced Accuracy": B_Accuracy,
377                    "ROC AUC": ROC_AUC,
378                    "F1 Score": F1,
379                    self.custom_metric.__name__: CUSTOM_METRIC,
380                    "Time Taken": TIME,
381                }
382            )
383        scores = scores.sort_values(by="Accuracy", ascending=False).set_index(
384            "Model"
385        )
386
387        if self.predictions:
388            predictions_df = pd.DataFrame.from_dict(predictions)
389        return scores, predictions_df if self.predictions is True else scores

Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.

def provide_models(self, X_train, X_test, y_train, y_test):
391    def provide_models(self, X_train, X_test, y_train, y_test):
392        """
393        This function returns all the model objects trained in fit function.
394        If fit is not called already, then we call fit and then return the models.
395        Parameters
396        ----------
397        X_train : array-like,
398            Training vectors, where rows is the number of samples
399            and columns is the number of features.
400        X_test : array-like,
401            Testing vectors, where rows is the number of samples
402            and columns is the number of features.
403        y_train : array-like,
404            Training vectors, where rows is the number of samples
405            and columns is the number of features.
406        y_test : array-like,
407            Testing vectors, where rows is the number of samples
408            and columns is the number of features.
409        Returns
410        -------
411        models: dict-object,
412            Returns a dictionary with each model pipeline as value
413            with key as name of models.
414        """
415        if len(self.models.keys()) == 0:
416            self.fit(X_train, X_test, y_train, y_test)
417
418        return self.models

This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.

def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.custom.custom.Custom
obj
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
sklearn.base.ClassifierMixin
score
class LazyDeepRegressor(nnetsauce.custom.custom.Custom, sklearn.base.RegressorMixin):
 85class LazyDeepRegressor(Custom, RegressorMixin):
 86    """
 87    This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's CustomRegressor
 88    Parameters
 89    ----------
 90    verbose : int, optional (default=0)
 91        For the liblinear and lbfgs solvers set verbose to any positive
 92        number for verbosity.
 93    ignore_warnings : bool, optional (default=True)
 94        When set to True, the warning related to algorigms that are not able to run are ignored.
 95    custom_metric : function, optional (default=None)
 96        When function is provided, models are evaluated based on the custom evaluation metric provided.
 97    prediction : bool, optional (default=False)
 98        When set to True, the predictions of all the models models are returned as dataframe.
 99    regressors : list, optional (default="all")
100        When function is provided, trains the chosen regressor(s).
101    n_jobs : int, when possible, run in parallel
102
103    Examples
104    --------
105    >>> from lazypredict.Supervised import LazyRegressor
106    >>> from sklearn import datasets
107    >>> from sklearn.utils import shuffle
108    >>> import numpy as np
109
110    >>> diabetes = datasets.load_diabetes()
111    >>> X, y = shuffle(diabetes.data, diabetes.target, random_state=13)
112    >>> X = X.astype(np.float32)
113
114    >>> offset = int(X.shape[0] * 0.9)
115    >>> X_train, y_train = X[:offset], y[:offset]
116    >>> X_test, y_test = X[offset:], y[offset:]
117
118    >>> reg = LazyDeepRegressor(verbose=0, ignore_warnings=False, custom_metric=None)
119    >>> models, predictions = reg.fit(X_train, X_test, y_train, y_test)
120    >>> model_dictionary = reg.provide_models(X_train, X_test, y_train, y_test)
121    >>> print(models)
122    """
123
124    def __init__(
125        self,
126        verbose=0,
127        ignore_warnings=True,
128        custom_metric=None,
129        predictions=False,
130        random_state=42,
131        regressors="all",
132        n_jobs=None,
133        # Defining depth
134        n_layers=3,
135        # CustomRegressor attributes
136        obj=None,
137        n_hidden_features=5,
138        activation_name="relu",
139        a=0.01,
140        nodes_sim="sobol",
141        bias=True,
142        dropout=0,
143        direct_link=True,
144        n_clusters=2,
145        cluster_encode=True,
146        type_clust="kmeans",
147        type_scaling=("std", "std", "std"),
148        col_sample=1,
149        row_sample=1,
150        seed=123,
151        backend="cpu",
152    ):
153        self.verbose = verbose
154        self.ignore_warnings = ignore_warnings
155        self.custom_metric = custom_metric
156        self.predictions = predictions
157        self.models = {}
158        self.random_state = random_state
159        self.regressors = regressors
160        self.n_layers = n_layers - 1
161        self.n_jobs = n_jobs
162        super().__init__(
163            obj=obj,
164            n_hidden_features=n_hidden_features,
165            activation_name=activation_name,
166            a=a,
167            nodes_sim=nodes_sim,
168            bias=bias,
169            dropout=dropout,
170            direct_link=direct_link,
171            n_clusters=n_clusters,
172            cluster_encode=cluster_encode,
173            type_clust=type_clust,
174            type_scaling=type_scaling,
175            col_sample=col_sample,
176            row_sample=row_sample,
177            seed=seed,
178            backend=backend,
179        )
180
181    def fit(self, X_train, X_test, y_train, y_test):
182        """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.
183        Parameters
184        ----------
185        X_train : array-like,
186            Training vectors, where rows is the number of samples
187            and columns is the number of features.
188        X_test : array-like,
189            Testing vectors, where rows is the number of samples
190            and columns is the number of features.
191        y_train : array-like,
192            Training vectors, where rows is the number of samples
193            and columns is the number of features.
194        y_test : array-like,
195            Testing vectors, where rows is the number of samples
196            and columns is the number of features.
197        Returns
198        -------
199        scores : Pandas DataFrame
200            Returns metrics of all the models in a Pandas DataFrame.
201        predictions : Pandas DataFrame
202            Returns predictions of all the models in a Pandas DataFrame.
203        """
204        R2 = []
205        ADJR2 = []
206        RMSE = []
207        # WIN = []
208        names = []
209        TIME = []
210        predictions = {}
211
212        if self.custom_metric:
213            CUSTOM_METRIC = []
214
215        if isinstance(X_train, np.ndarray):
216            X_train = pd.DataFrame(X_train)
217            X_test = pd.DataFrame(X_test)
218
219        numeric_features = X_train.select_dtypes(include=[np.number]).columns
220        categorical_features = X_train.select_dtypes(include=["object"]).columns
221
222        categorical_low, categorical_high = get_card_split(
223            X_train, categorical_features
224        )
225
226        if self.regressors == "all":
227            self.regressors = REGRESSORS
228        else:
229            try:
230                temp_list = []
231                for regressor in self.regressors:
232                    full_name = (regressor.__name__, regressor)
233                    temp_list.append(full_name)
234                self.regressors = temp_list
235            except Exception as exception:
236                print(exception)
237                print("Invalid Regressor(s)")
238
239        for name, model in tqdm(self.regressors):  # do parallel exec
240            start = time.time()
241            try:
242                if "random_state" in model().get_params().keys():
243                    layer_regr = CustomRegressor(
244                        obj=model(random_state=self.random_state),
245                        n_hidden_features=self.n_hidden_features,
246                        activation_name=self.activation_name,
247                        a=self.a,
248                        nodes_sim=self.nodes_sim,
249                        bias=self.bias,
250                        dropout=self.dropout,
251                        direct_link=self.direct_link,
252                        n_clusters=self.n_clusters,
253                        cluster_encode=self.cluster_encode,
254                        type_clust=self.type_clust,
255                        type_scaling=self.type_scaling,
256                        col_sample=self.col_sample,
257                        row_sample=self.row_sample,
258                        seed=self.seed,
259                        backend=self.backend,
260                    )
261                else:
262                    layer_regr = CustomRegressor(
263                        obj=model(),
264                        n_hidden_features=self.n_hidden_features,
265                        activation_name=self.activation_name,
266                        a=self.a,
267                        nodes_sim=self.nodes_sim,
268                        bias=self.bias,
269                        dropout=self.dropout,
270                        direct_link=self.direct_link,
271                        n_clusters=self.n_clusters,
272                        cluster_encode=self.cluster_encode,
273                        type_clust=self.type_clust,
274                        type_scaling=self.type_scaling,
275                        col_sample=self.col_sample,
276                        row_sample=self.row_sample,
277                        seed=self.seed,
278                        backend=self.backend,
279                    )
280
281                layer_regr.fit(X_train, y_train)
282
283                for _ in range(self.n_layers):
284                    layer_regr = deepcopy(
285                        CustomRegressor(
286                            obj=layer_regr,
287                            n_hidden_features=self.n_hidden_features,
288                            activation_name=self.activation_name,
289                            a=self.a,
290                            nodes_sim=self.nodes_sim,
291                            bias=self.bias,
292                            dropout=self.dropout,
293                            direct_link=self.direct_link,
294                            n_clusters=self.n_clusters,
295                            cluster_encode=self.cluster_encode,
296                            type_clust=self.type_clust,
297                            type_scaling=self.type_scaling,
298                            col_sample=self.col_sample,
299                            row_sample=self.row_sample,
300                            seed=self.seed,
301                            backend=self.backend,
302                        )
303                    )
304
305                    # layer_regr.fit(X_train, y_train)
306
307                layer_regr.fit(X_train, y_train)
308
309                self.models[name] = layer_regr
310                y_pred = layer_regr.predict(X_test)
311
312                r_squared = r2_score(y_test, y_pred)
313                adj_rsquared = adjusted_rsquared(
314                    r_squared, X_test.shape[0], X_test.shape[1]
315                )
316                rmse = mean_squared_error(y_test, y_pred, squared=False)
317
318                names.append(name)
319                R2.append(r_squared)
320                ADJR2.append(adj_rsquared)
321                RMSE.append(rmse)
322                TIME.append(time.time() - start)
323
324                if self.custom_metric:
325                    custom_metric = self.custom_metric(y_test, y_pred)
326                    CUSTOM_METRIC.append(custom_metric)
327
328                if self.verbose > 0:
329                    scores_verbose = {
330                        "Model": name,
331                        "R-Squared": r_squared,
332                        "Adjusted R-Squared": adj_rsquared,
333                        "RMSE": rmse,
334                        "Time taken": time.time() - start,
335                    }
336
337                    if self.custom_metric:
338                        scores_verbose[self.custom_metric.__name__] = (
339                            custom_metric
340                        )
341
342                    print(scores_verbose)
343                if self.predictions:
344                    predictions[name] = y_pred
345            except Exception as exception:
346                if self.ignore_warnings is False:
347                    print(name + " model failed to execute")
348                    print(exception)
349
350        scores = {
351            "Model": names,
352            "Adjusted R-Squared": ADJR2,
353            "R-Squared": R2,
354            "RMSE": RMSE,
355            "Time Taken": TIME,
356        }
357
358        if self.custom_metric:
359            scores[self.custom_metric.__name__] = CUSTOM_METRIC
360
361        scores = pd.DataFrame(scores)
362        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
363            "Model"
364        )
365
366        if self.predictions:
367            predictions_df = pd.DataFrame.from_dict(predictions)
368        return scores, predictions_df if self.predictions is True else scores
369
370    def provide_models(self, X_train, X_test, y_train, y_test):
371        """
372        This function returns all the model objects trained in fit function.
373        If fit is not called already, then we call fit and then return the models.
374        Parameters
375        ----------
376        X_train : array-like,
377            Training vectors, where rows is the number of samples
378            and columns is the number of features.
379        X_test : array-like,
380            Testing vectors, where rows is the number of samples
381            and columns is the number of features.
382        y_train : array-like,
383            Training vectors, where rows is the number of samples
384            and columns is the number of features.
385        y_test : array-like,
386            Testing vectors, where rows is the number of samples
387            and columns is the number of features.
388        Returns
389        -------
390        models: dict-object,
391            Returns a dictionary with each model pipeline as value
392            with key as name of models.
393        """
394        if len(self.models.keys()) == 0:
395            self.fit(X_train, X_test, y_train, y_test)
396
397        return self.models

This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's CustomRegressor

Parameters

verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. prediction : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. regressors : list, optional (default="all") When function is provided, trains the chosen regressor(s). n_jobs : int, when possible, run in parallel

Examples

>>> from lazypredict.Supervised import LazyRegressor
>>> from sklearn import datasets
>>> from sklearn.utils import shuffle
>>> import numpy as np
>>> diabetes = datasets.load_diabetes()
>>> X, y = shuffle(diabetes.data, diabetes.target, random_state=13)
>>> X = X.astype(np.float32)
>>> offset = int(X.shape[0] * 0.9)
>>> X_train, y_train = X[:offset], y[:offset]
>>> X_test, y_test = X[offset:], y[offset:]
>>> reg = LazyDeepRegressor(verbose=0, ignore_warnings=False, custom_metric=None)
>>> models, predictions = reg.fit(X_train, X_test, y_train, y_test)
>>> model_dictionary = reg.provide_models(X_train, X_test, y_train, y_test)
>>> print(models)
LazyDeepRegressor( verbose=0, ignore_warnings=True, custom_metric=None, predictions=False, random_state=42, regressors='all', n_jobs=None, n_layers=3, obj=None, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
124    def __init__(
125        self,
126        verbose=0,
127        ignore_warnings=True,
128        custom_metric=None,
129        predictions=False,
130        random_state=42,
131        regressors="all",
132        n_jobs=None,
133        # Defining depth
134        n_layers=3,
135        # CustomRegressor attributes
136        obj=None,
137        n_hidden_features=5,
138        activation_name="relu",
139        a=0.01,
140        nodes_sim="sobol",
141        bias=True,
142        dropout=0,
143        direct_link=True,
144        n_clusters=2,
145        cluster_encode=True,
146        type_clust="kmeans",
147        type_scaling=("std", "std", "std"),
148        col_sample=1,
149        row_sample=1,
150        seed=123,
151        backend="cpu",
152    ):
153        self.verbose = verbose
154        self.ignore_warnings = ignore_warnings
155        self.custom_metric = custom_metric
156        self.predictions = predictions
157        self.models = {}
158        self.random_state = random_state
159        self.regressors = regressors
160        self.n_layers = n_layers - 1
161        self.n_jobs = n_jobs
162        super().__init__(
163            obj=obj,
164            n_hidden_features=n_hidden_features,
165            activation_name=activation_name,
166            a=a,
167            nodes_sim=nodes_sim,
168            bias=bias,
169            dropout=dropout,
170            direct_link=direct_link,
171            n_clusters=n_clusters,
172            cluster_encode=cluster_encode,
173            type_clust=type_clust,
174            type_scaling=type_scaling,
175            col_sample=col_sample,
176            row_sample=row_sample,
177            seed=seed,
178            backend=backend,
179        )
verbose
ignore_warnings
custom_metric
predictions
models
random_state
regressors
n_layers
n_jobs
def fit(self, X_train, X_test, y_train, y_test):
181    def fit(self, X_train, X_test, y_train, y_test):
182        """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.
183        Parameters
184        ----------
185        X_train : array-like,
186            Training vectors, where rows is the number of samples
187            and columns is the number of features.
188        X_test : array-like,
189            Testing vectors, where rows is the number of samples
190            and columns is the number of features.
191        y_train : array-like,
192            Training vectors, where rows is the number of samples
193            and columns is the number of features.
194        y_test : array-like,
195            Testing vectors, where rows is the number of samples
196            and columns is the number of features.
197        Returns
198        -------
199        scores : Pandas DataFrame
200            Returns metrics of all the models in a Pandas DataFrame.
201        predictions : Pandas DataFrame
202            Returns predictions of all the models in a Pandas DataFrame.
203        """
204        R2 = []
205        ADJR2 = []
206        RMSE = []
207        # WIN = []
208        names = []
209        TIME = []
210        predictions = {}
211
212        if self.custom_metric:
213            CUSTOM_METRIC = []
214
215        if isinstance(X_train, np.ndarray):
216            X_train = pd.DataFrame(X_train)
217            X_test = pd.DataFrame(X_test)
218
219        numeric_features = X_train.select_dtypes(include=[np.number]).columns
220        categorical_features = X_train.select_dtypes(include=["object"]).columns
221
222        categorical_low, categorical_high = get_card_split(
223            X_train, categorical_features
224        )
225
226        if self.regressors == "all":
227            self.regressors = REGRESSORS
228        else:
229            try:
230                temp_list = []
231                for regressor in self.regressors:
232                    full_name = (regressor.__name__, regressor)
233                    temp_list.append(full_name)
234                self.regressors = temp_list
235            except Exception as exception:
236                print(exception)
237                print("Invalid Regressor(s)")
238
239        for name, model in tqdm(self.regressors):  # do parallel exec
240            start = time.time()
241            try:
242                if "random_state" in model().get_params().keys():
243                    layer_regr = CustomRegressor(
244                        obj=model(random_state=self.random_state),
245                        n_hidden_features=self.n_hidden_features,
246                        activation_name=self.activation_name,
247                        a=self.a,
248                        nodes_sim=self.nodes_sim,
249                        bias=self.bias,
250                        dropout=self.dropout,
251                        direct_link=self.direct_link,
252                        n_clusters=self.n_clusters,
253                        cluster_encode=self.cluster_encode,
254                        type_clust=self.type_clust,
255                        type_scaling=self.type_scaling,
256                        col_sample=self.col_sample,
257                        row_sample=self.row_sample,
258                        seed=self.seed,
259                        backend=self.backend,
260                    )
261                else:
262                    layer_regr = CustomRegressor(
263                        obj=model(),
264                        n_hidden_features=self.n_hidden_features,
265                        activation_name=self.activation_name,
266                        a=self.a,
267                        nodes_sim=self.nodes_sim,
268                        bias=self.bias,
269                        dropout=self.dropout,
270                        direct_link=self.direct_link,
271                        n_clusters=self.n_clusters,
272                        cluster_encode=self.cluster_encode,
273                        type_clust=self.type_clust,
274                        type_scaling=self.type_scaling,
275                        col_sample=self.col_sample,
276                        row_sample=self.row_sample,
277                        seed=self.seed,
278                        backend=self.backend,
279                    )
280
281                layer_regr.fit(X_train, y_train)
282
283                for _ in range(self.n_layers):
284                    layer_regr = deepcopy(
285                        CustomRegressor(
286                            obj=layer_regr,
287                            n_hidden_features=self.n_hidden_features,
288                            activation_name=self.activation_name,
289                            a=self.a,
290                            nodes_sim=self.nodes_sim,
291                            bias=self.bias,
292                            dropout=self.dropout,
293                            direct_link=self.direct_link,
294                            n_clusters=self.n_clusters,
295                            cluster_encode=self.cluster_encode,
296                            type_clust=self.type_clust,
297                            type_scaling=self.type_scaling,
298                            col_sample=self.col_sample,
299                            row_sample=self.row_sample,
300                            seed=self.seed,
301                            backend=self.backend,
302                        )
303                    )
304
305                    # layer_regr.fit(X_train, y_train)
306
307                layer_regr.fit(X_train, y_train)
308
309                self.models[name] = layer_regr
310                y_pred = layer_regr.predict(X_test)
311
312                r_squared = r2_score(y_test, y_pred)
313                adj_rsquared = adjusted_rsquared(
314                    r_squared, X_test.shape[0], X_test.shape[1]
315                )
316                rmse = mean_squared_error(y_test, y_pred, squared=False)
317
318                names.append(name)
319                R2.append(r_squared)
320                ADJR2.append(adj_rsquared)
321                RMSE.append(rmse)
322                TIME.append(time.time() - start)
323
324                if self.custom_metric:
325                    custom_metric = self.custom_metric(y_test, y_pred)
326                    CUSTOM_METRIC.append(custom_metric)
327
328                if self.verbose > 0:
329                    scores_verbose = {
330                        "Model": name,
331                        "R-Squared": r_squared,
332                        "Adjusted R-Squared": adj_rsquared,
333                        "RMSE": rmse,
334                        "Time taken": time.time() - start,
335                    }
336
337                    if self.custom_metric:
338                        scores_verbose[self.custom_metric.__name__] = (
339                            custom_metric
340                        )
341
342                    print(scores_verbose)
343                if self.predictions:
344                    predictions[name] = y_pred
345            except Exception as exception:
346                if self.ignore_warnings is False:
347                    print(name + " model failed to execute")
348                    print(exception)
349
350        scores = {
351            "Model": names,
352            "Adjusted R-Squared": ADJR2,
353            "R-Squared": R2,
354            "RMSE": RMSE,
355            "Time Taken": TIME,
356        }
357
358        if self.custom_metric:
359            scores[self.custom_metric.__name__] = CUSTOM_METRIC
360
361        scores = pd.DataFrame(scores)
362        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
363            "Model"
364        )
365
366        if self.predictions:
367            predictions_df = pd.DataFrame.from_dict(predictions)
368        return scores, predictions_df if self.predictions is True else scores

Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.

def provide_models(self, X_train, X_test, y_train, y_test):
370    def provide_models(self, X_train, X_test, y_train, y_test):
371        """
372        This function returns all the model objects trained in fit function.
373        If fit is not called already, then we call fit and then return the models.
374        Parameters
375        ----------
376        X_train : array-like,
377            Training vectors, where rows is the number of samples
378            and columns is the number of features.
379        X_test : array-like,
380            Testing vectors, where rows is the number of samples
381            and columns is the number of features.
382        y_train : array-like,
383            Training vectors, where rows is the number of samples
384            and columns is the number of features.
385        y_test : array-like,
386            Testing vectors, where rows is the number of samples
387            and columns is the number of features.
388        Returns
389        -------
390        models: dict-object,
391            Returns a dictionary with each model pipeline as value
392            with key as name of models.
393        """
394        if len(self.models.keys()) == 0:
395            self.fit(X_train, X_test, y_train, y_test)
396
397        return self.models

This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.

def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.custom.custom.Custom
obj
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
sklearn.base.RegressorMixin
score
class LazyDeepMTS(nnetsauce.MTS):
 91class LazyDeepMTS(MTS):
 92    """
 93    This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's MTS
 94    Parameters
 95    ----------
 96    verbose : int, optional (default=0)
 97        For the liblinear and lbfgs solvers set verbose to any positive
 98        number for verbosity.
 99    ignore_warnings : bool, optional (default=True)
100        When set to True, the warning related to algorigms that are not able to run are ignored.
101    custom_metric : function, optional (default=None)
102        When function is provided, models are evaluated based on the custom evaluation metric provided.
103    prediction : bool, optional (default=False)
104        When set to True, the predictions of all the models models are returned as dataframe.
105    regressors : list, optional (default="all")
106        When function is provided, trains the chosen regressor(s).
107
108    Examples
109    --------
110
111    """
112
113    def __init__(
114        self,
115        verbose=0,
116        ignore_warnings=True,
117        custom_metric=None,
118        predictions=False,
119        random_state=42,
120        regressors="all",
121        preprocess=False,
122        # Defining depth
123        n_layers=3,
124        # MTS attributes
125        obj=None,
126        n_hidden_features=5,
127        activation_name="relu",
128        a=0.01,
129        nodes_sim="sobol",
130        bias=True,
131        dropout=0,
132        direct_link=True,
133        n_clusters=2,
134        cluster_encode=True,
135        type_clust="kmeans",
136        type_scaling=("std", "std", "std"),
137        lags=1,
138        replications=None,
139        kernel=None,
140        agg="mean",
141        seed=123,
142        backend="cpu",
143        show_progress=False,
144    ):
145        self.verbose = verbose
146        self.ignore_warnings = ignore_warnings
147        self.custom_metric = custom_metric
148        self.predictions = predictions
149        self.models = {}
150        self.random_state = random_state
151        self.regressors = regressors
152        self.preprocess = preprocess
153        self.n_layers = n_layers
154        super().__init__(
155            obj=obj,
156            n_hidden_features=n_hidden_features,
157            activation_name=activation_name,
158            a=a,
159            nodes_sim=nodes_sim,
160            bias=bias,
161            dropout=dropout,
162            direct_link=direct_link,
163            n_clusters=n_clusters,
164            cluster_encode=cluster_encode,
165            type_clust=type_clust,
166            type_scaling=type_scaling,
167            seed=seed,
168            backend=backend,
169            lags=lags,
170            replications=replications,
171            kernel=kernel,
172            agg=agg,
173            show_progress=show_progress,
174        )
175
176    def fit(self, X_train, X_test, xreg=None, new_xreg=None, **kwargs):
177        """Fit Regression algorithms to X_train, predict and score on X_test.
178        Parameters
179        ----------
180        X_train : array-like,
181            Training vectors, where rows is the number of samples
182            and columns is the number of features.
183        X_test : array-like,
184            Testing vectors, where rows is the number of samples
185            and columns is the number of features.
186        Returns
187        -------
188        scores : Pandas DataFrame
189            Returns metrics of all the models in a Pandas DataFrame.
190        predictions : Pandas DataFrame
191            Returns predictions of all the models in a Pandas DataFrame.
192        """
193        R2 = []
194        ADJR2 = []
195        ME = []
196        MPL = []
197        RMSE = []
198        MAE = []
199        MPE = []
200        MAPE = []
201
202        # WIN = []
203        names = []
204        TIME = []
205        predictions = {}
206
207        if self.custom_metric:
208            CUSTOM_METRIC = []
209
210        if isinstance(X_train, np.ndarray):
211            X_train = pd.DataFrame(X_train)
212            X_test = pd.DataFrame(X_test)
213
214        X_train = convert_df_to_numeric(X_train)
215        X_test = convert_df_to_numeric(X_test)
216
217        numeric_features = X_train.select_dtypes(include=[np.number]).columns
218        categorical_features = X_train.select_dtypes(include=["object"]).columns
219
220        categorical_low, categorical_high = get_card_split(
221            X_train, categorical_features
222        )
223
224        if self.preprocess:
225            preprocessor = ColumnTransformer(
226                transformers=[
227                    ("numeric", numeric_transformer, numeric_features),
228                    (
229                        "categorical_low",
230                        categorical_transformer_low,
231                        categorical_low,
232                    ),
233                    (
234                        "categorical_high",
235                        categorical_transformer_high,
236                        categorical_high,
237                    ),
238                ]
239            )
240
241        if self.regressors == "all":
242            self.regressors = REGRESSORSDEEPMTS
243        else:
244            try:
245                temp_list = []
246                for regressor in self.regressors:
247                    full_name = (regressor.__name__, regressor)
248                    temp_list.append(full_name)
249                self.regressors = temp_list
250            except Exception as exception:
251                print(exception)
252                print("Invalid Regressor(s)")
253
254        if self.preprocess is True:
255            for name, model in tqdm(self.regressors):  # do parallel exec
256                start = time.time()
257                try:
258                    if "random_state" in model().get_params().keys():
259                        pipe = Pipeline(
260                            steps=[
261                                ("preprocessor", preprocessor),
262                                (
263                                    "regressor",
264                                    DeepMTS(
265                                        obj=model(
266                                            random_state=self.random_state,
267                                            **kwargs
268                                        ),
269                                        n_layers=self.n_layers,
270                                        n_hidden_features=self.n_hidden_features,
271                                        activation_name=self.activation_name,
272                                        a=self.a,
273                                        nodes_sim=self.nodes_sim,
274                                        bias=self.bias,
275                                        dropout=self.dropout,
276                                        direct_link=self.direct_link,
277                                        n_clusters=self.n_clusters,
278                                        cluster_encode=self.cluster_encode,
279                                        type_clust=self.type_clust,
280                                        type_scaling=self.type_scaling,
281                                        lags=self.lags,
282                                        replications=self.replications,
283                                        kernel=self.kernel,
284                                        agg=self.agg,
285                                        seed=self.seed,
286                                        backend=self.backend,
287                                        show_progress=self.show_progress,
288                                    ),
289                                ),
290                            ]
291                        )
292                    else:
293                        pipe = Pipeline(
294                            steps=[
295                                ("preprocessor", preprocessor),
296                                (
297                                    "regressor",
298                                    DeepMTS(
299                                        obj=model(**kwargs),
300                                        n_layers=self.n_layers,
301                                        n_hidden_features=self.n_hidden_features,
302                                        activation_name=self.activation_name,
303                                        a=self.a,
304                                        nodes_sim=self.nodes_sim,
305                                        bias=self.bias,
306                                        dropout=self.dropout,
307                                        direct_link=self.direct_link,
308                                        n_clusters=self.n_clusters,
309                                        cluster_encode=self.cluster_encode,
310                                        type_clust=self.type_clust,
311                                        type_scaling=self.type_scaling,
312                                        lags=self.lags,
313                                        replications=self.replications,
314                                        kernel=self.kernel,
315                                        agg=self.agg,
316                                        seed=self.seed,
317                                        backend=self.backend,
318                                        show_progress=self.show_progress,
319                                    ),
320                                ),
321                            ]
322                        )
323
324                    pipe.fit(X_train, **kwargs)
325                    # pipe.fit(X_train, xreg=xreg)
326
327                    self.models[name] = pipe
328                    if xreg is not None:
329                        assert (
330                            new_xreg is not None
331                        ), "xreg and new_xreg must be provided"
332                    # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg)
333                    X_pred = pipe["regressor"].predict(
334                        h=X_test.shape[0], **kwargs
335                    )
336                    rmse = mean_squared_error(X_test, X_pred, squared=False)
337                    mae = mean_absolute_error(X_test, X_pred)
338                    mpl = mean_pinball_loss(X_test, X_pred)
339
340                    names.append(name)
341                    RMSE.append(rmse)
342                    MAE.append(mae)
343                    MPL.append(mpl)
344                    TIME.append(time.time() - start)
345
346                    if self.custom_metric:
347                        custom_metric = self.custom_metric(X_test, X_pred)
348                        CUSTOM_METRIC.append(custom_metric)
349
350                    if self.verbose > 0:
351                        scores_verbose = {
352                            "Model": name,
353                            # "R-Squared": r_squared,
354                            # "Adjusted R-Squared": adj_rsquared,
355                            "RMSE": rmse,
356                            "MAE": mae,
357                            "MPL": mpl,
358                            # "MPE": mpe,
359                            # "MAPE": mape,
360                            "Time taken": time.time() - start,
361                        }
362
363                        if self.custom_metric:
364                            scores_verbose[self.custom_metric.__name__] = (
365                                custom_metric
366                            )
367
368                        print(scores_verbose)
369                    if self.predictions:
370                        predictions[name] = X_pred
371                except Exception as exception:
372                    if self.ignore_warnings is False:
373                        print(name + " model failed to execute")
374                        print(exception)
375
376        else:  # no preprocessing
377            for name, model in tqdm(self.regressors):  # do parallel exec
378                start = time.time()
379                try:
380                    if "random_state" in model().get_params().keys():
381                        pipe = DeepMTS(
382                            obj=model(random_state=self.random_state, **kwargs),
383                            n_hidden_features=self.n_hidden_features,
384                            activation_name=self.activation_name,
385                            a=self.a,
386                            nodes_sim=self.nodes_sim,
387                            bias=self.bias,
388                            dropout=self.dropout,
389                            direct_link=self.direct_link,
390                            n_clusters=self.n_clusters,
391                            cluster_encode=self.cluster_encode,
392                            type_clust=self.type_clust,
393                            type_scaling=self.type_scaling,
394                            lags=self.lags,
395                            replications=self.replications,
396                            kernel=self.kernel,
397                            agg=self.agg,
398                            seed=self.seed,
399                            backend=self.backend,
400                            show_progress=self.show_progress,
401                        )
402                    else:
403                        pipe = DeepMTS(
404                            obj=model(**kwargs),
405                            n_hidden_features=self.n_hidden_features,
406                            activation_name=self.activation_name,
407                            a=self.a,
408                            nodes_sim=self.nodes_sim,
409                            bias=self.bias,
410                            dropout=self.dropout,
411                            direct_link=self.direct_link,
412                            n_clusters=self.n_clusters,
413                            cluster_encode=self.cluster_encode,
414                            type_clust=self.type_clust,
415                            type_scaling=self.type_scaling,
416                            lags=self.lags,
417                            replications=self.replications,
418                            kernel=self.kernel,
419                            agg=self.agg,
420                            seed=self.seed,
421                            backend=self.backend,
422                            show_progress=self.show_progress,
423                        )
424
425                    pipe.fit(X_train, **kwargs)
426                    # pipe.fit(X_train, xreg=xreg) # DO xreg like in `ahead`
427
428                    self.models[name] = pipe
429                    if xreg is not None:
430                        assert (
431                            new_xreg is not None
432                        ), "xreg and new_xreg must be provided"
433
434                    if self.preprocess is True:
435                        X_pred = pipe["regressor"].predict(
436                            h=X_test.shape[0], **kwargs
437                        )
438                    else:
439                        X_pred = pipe.predict(
440                            h=X_test.shape[0], **kwargs
441                        )  # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) ## DO xreg like in `ahead`
442
443                    rmse = mean_squared_error(X_test, X_pred, squared=False)
444                    mae = mean_absolute_error(X_test, X_pred)
445                    mpl = mean_pinball_loss(X_test, X_pred)
446
447                    names.append(name)
448                    RMSE.append(rmse)
449                    MAE.append(mae)
450                    MPL.append(mpl)
451                    TIME.append(time.time() - start)
452
453                    if self.custom_metric:
454                        custom_metric = self.custom_metric(X_test, X_pred)
455                        CUSTOM_METRIC.append(custom_metric)
456
457                    if self.verbose > 0:
458                        scores_verbose = {
459                            "Model": name,
460                            # "R-Squared": r_squared,
461                            # "Adjusted R-Squared": adj_rsquared,
462                            "RMSE": rmse,
463                            "MAE": mae,
464                            "MPL": mpl,
465                            # "MPE": mpe,
466                            # "MAPE": mape,
467                            "Time taken": time.time() - start,
468                        }
469
470                        if self.custom_metric:
471                            scores_verbose[self.custom_metric.__name__] = (
472                                custom_metric
473                            )
474
475                        print(scores_verbose)
476                    if self.predictions:
477                        predictions[name] = X_pred
478                except Exception as exception:
479                    if self.ignore_warnings is False:
480                        print(name + " model failed to execute")
481                        print(exception)
482
483        scores = {
484            "Model": names,
485            # "Adjusted R-Squared": ADJR2,
486            # "R-Squared": R2,
487            "RMSE": RMSE,
488            "MAE": MAE,
489            "MPL": MPL,
490            # "MPE": MPE,
491            # "MAPE": MAPE,
492            "Time Taken": TIME,
493        }
494
495        if self.custom_metric:
496            scores[self.custom_metric.__name__] = CUSTOM_METRIC
497
498        scores = pd.DataFrame(scores)
499        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
500            "Model"
501        )
502
503        if self.predictions:
504            predictions_df = pd.DataFrame.from_dict(predictions)
505        return scores, predictions_df if self.predictions is True else scores
506
507    def provide_models(self, X_train, X_test):
508        """
509        This function returns all the model objects trained in fit function.
510        If fit is not called already, then we call fit and then return the models.
511        Parameters
512        ----------
513        X_train : array-like,
514            Training vectors, where rows is the number of samples
515            and columns is the number of features.
516        X_test : array-like,
517            Testing vectors, where rows is the number of samples
518            and columns is the number of features.
519        Returns
520        -------
521        models: dict-object,
522            Returns a dictionary with each model pipeline as value
523            with key as name of models.
524        """
525        if len(self.models.keys()) == 0:
526            self.fit(X_train, X_test)
527
528        return self.models

This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's MTS

Parameters

verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. prediction : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. regressors : list, optional (default="all") When function is provided, trains the chosen regressor(s).

Examples

LazyDeepMTS( verbose=0, ignore_warnings=True, custom_metric=None, predictions=False, random_state=42, regressors='all', preprocess=False, n_layers=3, obj=None, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), lags=1, replications=None, kernel=None, agg='mean', seed=123, backend='cpu', show_progress=False)
113    def __init__(
114        self,
115        verbose=0,
116        ignore_warnings=True,
117        custom_metric=None,
118        predictions=False,
119        random_state=42,
120        regressors="all",
121        preprocess=False,
122        # Defining depth
123        n_layers=3,
124        # MTS attributes
125        obj=None,
126        n_hidden_features=5,
127        activation_name="relu",
128        a=0.01,
129        nodes_sim="sobol",
130        bias=True,
131        dropout=0,
132        direct_link=True,
133        n_clusters=2,
134        cluster_encode=True,
135        type_clust="kmeans",
136        type_scaling=("std", "std", "std"),
137        lags=1,
138        replications=None,
139        kernel=None,
140        agg="mean",
141        seed=123,
142        backend="cpu",
143        show_progress=False,
144    ):
145        self.verbose = verbose
146        self.ignore_warnings = ignore_warnings
147        self.custom_metric = custom_metric
148        self.predictions = predictions
149        self.models = {}
150        self.random_state = random_state
151        self.regressors = regressors
152        self.preprocess = preprocess
153        self.n_layers = n_layers
154        super().__init__(
155            obj=obj,
156            n_hidden_features=n_hidden_features,
157            activation_name=activation_name,
158            a=a,
159            nodes_sim=nodes_sim,
160            bias=bias,
161            dropout=dropout,
162            direct_link=direct_link,
163            n_clusters=n_clusters,
164            cluster_encode=cluster_encode,
165            type_clust=type_clust,
166            type_scaling=type_scaling,
167            seed=seed,
168            backend=backend,
169            lags=lags,
170            replications=replications,
171            kernel=kernel,
172            agg=agg,
173            show_progress=show_progress,
174        )
verbose
ignore_warnings
custom_metric
predictions
models
random_state
regressors
preprocess
n_layers
def fit(self, X_train, X_test, xreg=None, new_xreg=None, **kwargs):
176    def fit(self, X_train, X_test, xreg=None, new_xreg=None, **kwargs):
177        """Fit Regression algorithms to X_train, predict and score on X_test.
178        Parameters
179        ----------
180        X_train : array-like,
181            Training vectors, where rows is the number of samples
182            and columns is the number of features.
183        X_test : array-like,
184            Testing vectors, where rows is the number of samples
185            and columns is the number of features.
186        Returns
187        -------
188        scores : Pandas DataFrame
189            Returns metrics of all the models in a Pandas DataFrame.
190        predictions : Pandas DataFrame
191            Returns predictions of all the models in a Pandas DataFrame.
192        """
193        R2 = []
194        ADJR2 = []
195        ME = []
196        MPL = []
197        RMSE = []
198        MAE = []
199        MPE = []
200        MAPE = []
201
202        # WIN = []
203        names = []
204        TIME = []
205        predictions = {}
206
207        if self.custom_metric:
208            CUSTOM_METRIC = []
209
210        if isinstance(X_train, np.ndarray):
211            X_train = pd.DataFrame(X_train)
212            X_test = pd.DataFrame(X_test)
213
214        X_train = convert_df_to_numeric(X_train)
215        X_test = convert_df_to_numeric(X_test)
216
217        numeric_features = X_train.select_dtypes(include=[np.number]).columns
218        categorical_features = X_train.select_dtypes(include=["object"]).columns
219
220        categorical_low, categorical_high = get_card_split(
221            X_train, categorical_features
222        )
223
224        if self.preprocess:
225            preprocessor = ColumnTransformer(
226                transformers=[
227                    ("numeric", numeric_transformer, numeric_features),
228                    (
229                        "categorical_low",
230                        categorical_transformer_low,
231                        categorical_low,
232                    ),
233                    (
234                        "categorical_high",
235                        categorical_transformer_high,
236                        categorical_high,
237                    ),
238                ]
239            )
240
241        if self.regressors == "all":
242            self.regressors = REGRESSORSDEEPMTS
243        else:
244            try:
245                temp_list = []
246                for regressor in self.regressors:
247                    full_name = (regressor.__name__, regressor)
248                    temp_list.append(full_name)
249                self.regressors = temp_list
250            except Exception as exception:
251                print(exception)
252                print("Invalid Regressor(s)")
253
254        if self.preprocess is True:
255            for name, model in tqdm(self.regressors):  # do parallel exec
256                start = time.time()
257                try:
258                    if "random_state" in model().get_params().keys():
259                        pipe = Pipeline(
260                            steps=[
261                                ("preprocessor", preprocessor),
262                                (
263                                    "regressor",
264                                    DeepMTS(
265                                        obj=model(
266                                            random_state=self.random_state,
267                                            **kwargs
268                                        ),
269                                        n_layers=self.n_layers,
270                                        n_hidden_features=self.n_hidden_features,
271                                        activation_name=self.activation_name,
272                                        a=self.a,
273                                        nodes_sim=self.nodes_sim,
274                                        bias=self.bias,
275                                        dropout=self.dropout,
276                                        direct_link=self.direct_link,
277                                        n_clusters=self.n_clusters,
278                                        cluster_encode=self.cluster_encode,
279                                        type_clust=self.type_clust,
280                                        type_scaling=self.type_scaling,
281                                        lags=self.lags,
282                                        replications=self.replications,
283                                        kernel=self.kernel,
284                                        agg=self.agg,
285                                        seed=self.seed,
286                                        backend=self.backend,
287                                        show_progress=self.show_progress,
288                                    ),
289                                ),
290                            ]
291                        )
292                    else:
293                        pipe = Pipeline(
294                            steps=[
295                                ("preprocessor", preprocessor),
296                                (
297                                    "regressor",
298                                    DeepMTS(
299                                        obj=model(**kwargs),
300                                        n_layers=self.n_layers,
301                                        n_hidden_features=self.n_hidden_features,
302                                        activation_name=self.activation_name,
303                                        a=self.a,
304                                        nodes_sim=self.nodes_sim,
305                                        bias=self.bias,
306                                        dropout=self.dropout,
307                                        direct_link=self.direct_link,
308                                        n_clusters=self.n_clusters,
309                                        cluster_encode=self.cluster_encode,
310                                        type_clust=self.type_clust,
311                                        type_scaling=self.type_scaling,
312                                        lags=self.lags,
313                                        replications=self.replications,
314                                        kernel=self.kernel,
315                                        agg=self.agg,
316                                        seed=self.seed,
317                                        backend=self.backend,
318                                        show_progress=self.show_progress,
319                                    ),
320                                ),
321                            ]
322                        )
323
324                    pipe.fit(X_train, **kwargs)
325                    # pipe.fit(X_train, xreg=xreg)
326
327                    self.models[name] = pipe
328                    if xreg is not None:
329                        assert (
330                            new_xreg is not None
331                        ), "xreg and new_xreg must be provided"
332                    # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg)
333                    X_pred = pipe["regressor"].predict(
334                        h=X_test.shape[0], **kwargs
335                    )
336                    rmse = mean_squared_error(X_test, X_pred, squared=False)
337                    mae = mean_absolute_error(X_test, X_pred)
338                    mpl = mean_pinball_loss(X_test, X_pred)
339
340                    names.append(name)
341                    RMSE.append(rmse)
342                    MAE.append(mae)
343                    MPL.append(mpl)
344                    TIME.append(time.time() - start)
345
346                    if self.custom_metric:
347                        custom_metric = self.custom_metric(X_test, X_pred)
348                        CUSTOM_METRIC.append(custom_metric)
349
350                    if self.verbose > 0:
351                        scores_verbose = {
352                            "Model": name,
353                            # "R-Squared": r_squared,
354                            # "Adjusted R-Squared": adj_rsquared,
355                            "RMSE": rmse,
356                            "MAE": mae,
357                            "MPL": mpl,
358                            # "MPE": mpe,
359                            # "MAPE": mape,
360                            "Time taken": time.time() - start,
361                        }
362
363                        if self.custom_metric:
364                            scores_verbose[self.custom_metric.__name__] = (
365                                custom_metric
366                            )
367
368                        print(scores_verbose)
369                    if self.predictions:
370                        predictions[name] = X_pred
371                except Exception as exception:
372                    if self.ignore_warnings is False:
373                        print(name + " model failed to execute")
374                        print(exception)
375
376        else:  # no preprocessing
377            for name, model in tqdm(self.regressors):  # do parallel exec
378                start = time.time()
379                try:
380                    if "random_state" in model().get_params().keys():
381                        pipe = DeepMTS(
382                            obj=model(random_state=self.random_state, **kwargs),
383                            n_hidden_features=self.n_hidden_features,
384                            activation_name=self.activation_name,
385                            a=self.a,
386                            nodes_sim=self.nodes_sim,
387                            bias=self.bias,
388                            dropout=self.dropout,
389                            direct_link=self.direct_link,
390                            n_clusters=self.n_clusters,
391                            cluster_encode=self.cluster_encode,
392                            type_clust=self.type_clust,
393                            type_scaling=self.type_scaling,
394                            lags=self.lags,
395                            replications=self.replications,
396                            kernel=self.kernel,
397                            agg=self.agg,
398                            seed=self.seed,
399                            backend=self.backend,
400                            show_progress=self.show_progress,
401                        )
402                    else:
403                        pipe = DeepMTS(
404                            obj=model(**kwargs),
405                            n_hidden_features=self.n_hidden_features,
406                            activation_name=self.activation_name,
407                            a=self.a,
408                            nodes_sim=self.nodes_sim,
409                            bias=self.bias,
410                            dropout=self.dropout,
411                            direct_link=self.direct_link,
412                            n_clusters=self.n_clusters,
413                            cluster_encode=self.cluster_encode,
414                            type_clust=self.type_clust,
415                            type_scaling=self.type_scaling,
416                            lags=self.lags,
417                            replications=self.replications,
418                            kernel=self.kernel,
419                            agg=self.agg,
420                            seed=self.seed,
421                            backend=self.backend,
422                            show_progress=self.show_progress,
423                        )
424
425                    pipe.fit(X_train, **kwargs)
426                    # pipe.fit(X_train, xreg=xreg) # DO xreg like in `ahead`
427
428                    self.models[name] = pipe
429                    if xreg is not None:
430                        assert (
431                            new_xreg is not None
432                        ), "xreg and new_xreg must be provided"
433
434                    if self.preprocess is True:
435                        X_pred = pipe["regressor"].predict(
436                            h=X_test.shape[0], **kwargs
437                        )
438                    else:
439                        X_pred = pipe.predict(
440                            h=X_test.shape[0], **kwargs
441                        )  # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) ## DO xreg like in `ahead`
442
443                    rmse = mean_squared_error(X_test, X_pred, squared=False)
444                    mae = mean_absolute_error(X_test, X_pred)
445                    mpl = mean_pinball_loss(X_test, X_pred)
446
447                    names.append(name)
448                    RMSE.append(rmse)
449                    MAE.append(mae)
450                    MPL.append(mpl)
451                    TIME.append(time.time() - start)
452
453                    if self.custom_metric:
454                        custom_metric = self.custom_metric(X_test, X_pred)
455                        CUSTOM_METRIC.append(custom_metric)
456
457                    if self.verbose > 0:
458                        scores_verbose = {
459                            "Model": name,
460                            # "R-Squared": r_squared,
461                            # "Adjusted R-Squared": adj_rsquared,
462                            "RMSE": rmse,
463                            "MAE": mae,
464                            "MPL": mpl,
465                            # "MPE": mpe,
466                            # "MAPE": mape,
467                            "Time taken": time.time() - start,
468                        }
469
470                        if self.custom_metric:
471                            scores_verbose[self.custom_metric.__name__] = (
472                                custom_metric
473                            )
474
475                        print(scores_verbose)
476                    if self.predictions:
477                        predictions[name] = X_pred
478                except Exception as exception:
479                    if self.ignore_warnings is False:
480                        print(name + " model failed to execute")
481                        print(exception)
482
483        scores = {
484            "Model": names,
485            # "Adjusted R-Squared": ADJR2,
486            # "R-Squared": R2,
487            "RMSE": RMSE,
488            "MAE": MAE,
489            "MPL": MPL,
490            # "MPE": MPE,
491            # "MAPE": MAPE,
492            "Time Taken": TIME,
493        }
494
495        if self.custom_metric:
496            scores[self.custom_metric.__name__] = CUSTOM_METRIC
497
498        scores = pd.DataFrame(scores)
499        scores = scores.sort_values(by="RMSE", ascending=True).set_index(
500            "Model"
501        )
502
503        if self.predictions:
504            predictions_df = pd.DataFrame.from_dict(predictions)
505        return scores, predictions_df if self.predictions is True else scores

Fit Regression algorithms to X_train, predict and score on X_test.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.

def provide_models(self, X_train, X_test):
507    def provide_models(self, X_train, X_test):
508        """
509        This function returns all the model objects trained in fit function.
510        If fit is not called already, then we call fit and then return the models.
511        Parameters
512        ----------
513        X_train : array-like,
514            Training vectors, where rows is the number of samples
515            and columns is the number of features.
516        X_test : array-like,
517            Testing vectors, where rows is the number of samples
518            and columns is the number of features.
519        Returns
520        -------
521        models: dict-object,
522            Returns a dictionary with each model pipeline as value
523            with key as name of models.
524        """
525        if len(self.models.keys()) == 0:
526            self.fit(X_train, X_test)
527
528        return self.models

This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.

Parameters

X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.

Returns

models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.

def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_predict_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class MTS(nnetsauce.Base):
 23class MTS(Base):
 24    """Univariate and multivariate time series (MTS) forecasting with Quasi-Randomized networks (Work in progress /!\)
 25
 26    Parameters:
 27
 28        obj: object.
 29            any object containing a method fit (obj.fit()) and a method predict
 30            (obj.predict()).
 31
 32        n_hidden_features: int.
 33            number of nodes in the hidden layer.
 34
 35        activation_name: str.
 36            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'.
 37
 38        a: float.
 39            hyperparameter for 'prelu' or 'elu' activation function.
 40
 41        nodes_sim: str.
 42            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 43            'uniform'.
 44
 45        bias: boolean.
 46            indicates if the hidden layer contains a bias term (True) or not
 47            (False).
 48
 49        dropout: float.
 50            regularization parameter; (random) percentage of nodes dropped out
 51            of the training.
 52
 53        direct_link: boolean.
 54            indicates if the original predictors are included (True) in model's fitting or not (False).
 55
 56        n_clusters: int.
 57            number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering).
 58
 59        cluster_encode: bool.
 60            defines how the variable containing clusters is treated (default is one-hot)
 61            if `False`, then labels are used, without one-hot encoding.
 62
 63        type_clust: str.
 64            type of clustering method: currently k-means ('kmeans') or Gaussian
 65            Mixture Model ('gmm').
 66
 67        type_scaling: a tuple of 3 strings.
 68            scaling methods for inputs, hidden layer, and clustering respectively
 69            (and when relevant).
 70            Currently available: standardization ('std') or MinMax scaling ('minmax').
 71
 72        lags: int.
 73            number of lags used for each time series.
 74
 75        replications: int.
 76            number of replications (if needed, for predictive simulation). Default is 'None'.
 77
 78        kernel: str.
 79            the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'.
 80
 81        agg: str.
 82            either "mean" or "median" for simulation of bootstrap aggregating
 83
 84        seed: int.
 85            reproducibility seed for nodes_sim=='uniform' or predictive simulation.
 86
 87        backend: str.
 88            "cpu" or "gpu" or "tpu".
 89
 90        verbose: int.
 91            0: not printing; 1: printing
 92
 93        show_progress: bool.
 94            True: progress bar when fitting each series; False: no progress bar when fitting each series
 95
 96    Attributes:
 97
 98        fit_objs_: dict
 99            objects adjusted to each individual time series
100
101        y_: {array-like}
102            MTS responses (most recent observations first)
103
104        X_: {array-like}
105            MTS lags
106
107        xreg_: {array-like}
108            external regressors
109
110        y_means_: dict
111            a dictionary of each series mean values
112
113        preds_: {array-like}
114            successive model predictions
115
116        preds_std_: {array-like}
117            standard deviation around the predictions
118
119        return_std_: boolean
120            return uncertainty or not (set in predict)
121
122        df_: data frame
123            the input data frame, in case a data.frame is provided to `fit`
124
125    Examples:
126
127    Example 1:
128
129    ```python
130    import nnetsauce as ns
131    import numpy as np
132    from sklearn import linear_model
133    np.random.seed(123)
134
135    M = np.random.rand(10, 3)
136    M[:,0] = 10*M[:,0]
137    M[:,2] = 25*M[:,2]
138    print(M)
139
140    # Adjust Bayesian Ridge
141    regr4 = linear_model.BayesianRidge()
142    obj_MTS = ns.MTS(regr4, lags = 1, n_hidden_features=5)
143    obj_MTS.fit(M)
144    print(obj_MTS.predict())
145
146    # with credible intervals
147    print(obj_MTS.predict(return_std=True, level=80))
148
149    print(obj_MTS.predict(return_std=True, level=95))
150    ```
151
152    Example 2:
153
154    ```python
155    import nnetsauce as ns
156    import numpy as np
157    from sklearn import linear_model
158
159    dataset = {
160    'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'],
161    'series1' : [34, 30, 35.6, 33.3, 38.1],
162    'series2' : [4, 5.5, 5.6, 6.3, 5.1],
163    'series3' : [100, 100.5, 100.6, 100.2, 100.1]}
164    df = pd.DataFrame(dataset).set_index('date')
165    print(df)
166
167    # Adjust Bayesian Ridge
168    regr5 = linear_model.BayesianRidge()
169    obj_MTS = ns.MTS(regr5, lags = 1, n_hidden_features=5)
170    obj_MTS.fit(df)
171    print(obj_MTS.predict())
172
173    # with credible intervals
174    print(obj_MTS.predict(return_std=True, level=80))
175
176    print(obj_MTS.predict(return_std=True, level=95))
177    ```
178    """
179
180    # construct the object -----
181
182    def __init__(
183        self,
184        obj,
185        n_hidden_features=5,
186        activation_name="relu",
187        a=0.01,
188        nodes_sim="sobol",
189        bias=True,
190        dropout=0,
191        direct_link=True,
192        n_clusters=2,
193        cluster_encode=True,
194        type_clust="kmeans",
195        type_scaling=("std", "std", "std"),
196        lags=1,
197        type_pi="kde",
198        replications=None,
199        kernel=None,
200        agg="mean",
201        seed=123,
202        backend="cpu",
203        verbose=0,
204        show_progress=True,
205    ):
206        assert int(lags) == lags, "parameter 'lags' should be an integer"
207
208        super().__init__(
209            n_hidden_features=n_hidden_features,
210            activation_name=activation_name,
211            a=a,
212            nodes_sim=nodes_sim,
213            bias=bias,
214            dropout=dropout,
215            direct_link=direct_link,
216            n_clusters=n_clusters,
217            cluster_encode=cluster_encode,
218            type_clust=type_clust,
219            type_scaling=type_scaling,
220            seed=seed,
221            backend=backend,
222        )
223
224        self.obj = obj
225        self.n_series = None
226        self.lags = lags
227        self.type_pi = type_pi
228        self.replications = replications
229        self.kernel = kernel
230        self.agg = agg
231        self.verbose = verbose
232        self.show_progress = show_progress
233        self.series_names = None
234        self.input_dates = None
235        self.fit_objs_ = {}
236        self.y_ = None  # MTS responses (most recent observations first)
237        self.X_ = None  # MTS lags
238        self.xreg_ = None
239        self.y_means_ = {}
240        self.mean_ = None
241        self.upper_ = None
242        self.lower_ = None
243        self.output_dates_ = None
244        self.preds_std_ = []
245        self.alpha_ = None
246        self.return_std_ = None
247        self.df_ = None
248        self.residuals_ = []
249        self.residuals_sims_ = None
250        self.kde_ = None
251        self.sims_ = None
252
253    def fit(self, X, xreg=None, **kwargs):
254        """Fit MTS model to training data X, with optional regressors xreg
255
256        Parameters:
257
258            X: {array-like}, shape = [n_samples, n_features]
259                Training time series, where n_samples is the number
260                of samples and n_features is the number of features;
261                X must be in increasing order (most recent observations last)
262
263            xreg: {array-like}, shape = [n_samples, n_features_xreg]
264                Additional regressors to be passed to obj
265                xreg must be in increasing order (most recent observations last)
266
267            **kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity)
268
269        Returns:
270
271            self: object
272        """
273
274        if isinstance(X, pd.DataFrame) is False:
275            self.series_names = ["series" + str(i) for i in range(X.shape[1])]
276            X = pd.DataFrame(X, columns=self.series_names)
277        else:
278            X = copy.deepcopy(mo.convert_df_to_numeric(X))
279            self.series_names = list(X.columns.values)
280
281        self.df_ = X
282        X = X.values
283        self.input_dates = ts.compute_input_dates(self.df_)
284
285        try:
286            # multivariate time series
287            n, p = X.shape
288        except:
289            # univariate time series
290            n = X.shape[0]
291            p = 1
292
293        rep_1_n = np.repeat(1, n)
294
295        self.y_ = None
296        self.X_ = None
297        self.n_series = p
298        self.fit_objs_.clear()
299        self.y_means_.clear()
300        residuals_ = []
301        self.residuals_ = None
302        self.residuals_sims_ = None
303        self.kde_ = None
304        self.sims_ = None
305        self.scaled_Z_ = None
306        self.centered_y_is_ = []
307
308        if p > 1:
309            # multivariate time series
310            mts_input = ts.create_train_inputs(X[::-1], self.lags)
311        else:
312            # univariate time series
313            mts_input = ts.create_train_inputs(
314                X.reshape(-1, 1)[::-1], self.lags
315            )
316
317        self.y_ = mts_input[0]
318
319        self.X_ = mts_input[1]
320
321        if xreg is not None:
322            if isinstance(xreg, pd.DataFrame):
323                xreg = xreg.values
324
325            assert (
326                xreg.shape[0] == n
327            ), "'xreg' and 'X' must have the same number of observations"
328
329            self.xreg_ = xreg
330
331            xreg_input = ts.create_train_inputs(xreg[::-1], self.lags)
332
333            dummy_y, scaled_Z = self.cook_training_set(
334                y=rep_1_n,
335                X=mo.cbind(self.X_, xreg_input[1], backend=self.backend),
336            )
337
338        else:  # xreg is None
339            # avoids scaling X p times in the loop
340            dummy_y, scaled_Z = self.cook_training_set(y=rep_1_n, X=self.X_)
341
342        self.scaled_Z_ = scaled_Z
343
344        # loop on all the time series and adjust self.obj.fit
345        if self.verbose > 0:
346            print(
347                f"\n Adjusting {type(self.obj).__name__} to multivariate time series... \n "
348            )
349
350        if self.show_progress is True:
351            iterator = tqdm(range(p))
352        else:
353            iterator = range(p)
354
355        for i in iterator:
356            y_mean = np.mean(self.y_[:, i])
357            self.y_means_[i] = y_mean
358            centered_y_i = self.y_[:, i] - y_mean
359            self.centered_y_is_.append(centered_y_i)
360            self.obj.fit(X=scaled_Z, y=centered_y_i)
361            self.fit_objs_[i] = deepcopy(self.obj)
362            residuals_.append(
363                (centered_y_i - self.fit_objs_[i].predict(scaled_Z)).tolist()
364            )
365
366        self.residuals_ = np.asarray(residuals_).T
367
368        if self.replications != None and self.type_pi == "kde":
369            if self.verbose > 0:
370                print(f"\n Simulate residuals using {self.kernel} kernel... \n")
371            assert self.kernel in (
372                "gaussian",
373                "tophat",
374            ), "currently, 'kernel' must be either 'gaussian' or 'tophat'"
375            kernel_bandwidths = {"bandwidth": np.logspace(-6, 6, 150)}
376            grid = GridSearchCV(
377                KernelDensity(kernel=self.kernel, **kwargs),
378                param_grid=kernel_bandwidths,
379            )
380            grid.fit(self.residuals_)
381
382            if self.verbose > 0:
383                print(
384                    f"\n Best parameters for {self.kernel} kernel: {grid.best_params_} \n"
385                )
386
387            self.kde_ = grid.best_estimator_
388
389        return self
390
391    def predict(self, h=5, level=95, new_xreg=None, **kwargs):
392        """Forecast all the time series, h steps ahead
393
394        Parameters:
395
396            h: {integer}
397                Forecasting horizon
398
399            level: {integer}
400                Level of confidence (if obj has option 'return_std' and the
401                posterior is gaussian)
402
403            new_xreg: {array-like}, shape = [n_samples = h, n_new_xreg]
404                New values of additional (deterministic) regressors on horizon = h
405                new_xreg must be in increasing order (most recent observations last)
406
407            **kwargs: additional parameters to be passed to
408                    self.cook_test_set
409
410        Returns:
411
412            model predictions for horizon = h: {array-like}, data frame or tuple.
413            Standard deviation and prediction intervals are returned when
414            `obj.predict` can return standard deviation
415        """
416
417        self.output_dates_, frequency = ts.compute_output_dates(self.df_, h)
418
419        self.return_std_ = False  # do not remove (/!\)
420
421        self.mean_ = None  # do not remove (/!\)
422
423        self.mean_ = deepcopy(self.y_)  # do not remove (/!\)
424
425        self.lower_ = None
426
427        self.upper_ = None
428
429        self.sims_ = None
430
431        y_means_ = np.asarray([self.y_means_[i] for i in range(self.n_series)])
432
433        n_features = self.n_series * self.lags
434
435        self.alpha_ = 100 - level
436
437        if "return_std" in kwargs:
438            self.return_std_ = True
439            self.preds_std_ = []
440            pi_multiplier = norm.ppf(1 - self.alpha_ / 200)
441            DescribeResult = namedtuple(
442                "DescribeResult", ("mean", "lower", "upper")
443            )  # to be updated
444
445        if self.xreg_ is None:  # no external regressors
446            if self.kde_ != None and self.type_pi == "kde":
447                self.residuals_sims_ = tuple(
448                    self.kde_.sample(
449                        n_samples=h, random_state=self.seed + 100 * i
450                    )
451                    for i in tqdm(range(self.replications))
452                )
453
454            for _ in range(h):
455                new_obs = ts.reformat_response(self.mean_, self.lags)
456
457                new_X = new_obs.reshape(1, n_features)
458
459                cooked_new_X = self.cook_test_set(new_X, **kwargs)
460
461                if "return_std" in kwargs:
462                    self.preds_std_.append(
463                        [
464                            np.asarray(
465                                self.fit_objs_[i].predict(
466                                    cooked_new_X, return_std=True
467                                )[1]
468                            ).item()
469                            for i in range(self.n_series)
470                        ]
471                    )
472
473                predicted_cooked_new_X = np.asarray(
474                    [
475                        np.asarray(
476                            self.fit_objs_[i].predict(cooked_new_X)
477                        ).item()
478                        for i in range(self.n_series)
479                    ]
480                )
481
482                preds = np.asarray(y_means_ + predicted_cooked_new_X)
483
484                self.mean_ = mo.rbind(preds, self.mean_)
485
486        else:  # if self.xreg_ is not None: # with external regressors
487            assert (
488                new_xreg is not None
489            ), "'new_xreg' must be provided to predict()"
490
491            if isinstance(new_xreg, pd.DataFrame):
492                new_xreg = new_xreg.values
493
494            if self.kde_ is not None:
495                if self.verbose > 0:
496                    print("\n Obtain simulations for adjusted residuals... \n")
497                self.residuals_sims_ = tuple(
498                    self.kde_.sample(
499                        n_samples=h, random_state=self.seed + 100 * i
500                    )
501                    for i in tqdm(range(self.replications))
502                )
503
504            try:
505                n_obs_xreg, n_features_xreg = new_xreg.shape
506                assert (
507                    n_features_xreg == self.xreg_.shape[1]
508                ), "check number of inputs provided for 'new_xreg' (compare with self.xreg_.shape[1])"
509            except:
510                n_obs_xreg = new_xreg.shape  # one series
511
512            assert (
513                n_obs_xreg == h
514            ), "please provide values of regressors 'new_xreg' for the whole horizon 'h'"
515
516            n_features_xreg = n_features_xreg * self.lags
517
518            inv_new_xreg = mo.rbind(self.xreg_, new_xreg)[::-1]
519
520            for _ in range(h):
521                new_obs = ts.reformat_response(self.mean_, self.lags)
522
523                new_obs_xreg = ts.reformat_response(inv_new_xreg, self.lags)
524
525                new_X = np.concatenate(
526                    (new_obs, new_obs_xreg), axis=None
527                ).reshape(1, -1)
528
529                cooked_new_X = self.cook_test_set(new_X, **kwargs)
530
531                if "return_std" in kwargs:
532                    self.preds_std_.append(
533                        [
534                            np.asarray(
535                                self.fit_objs_[i].predict(
536                                    cooked_new_X, return_std=True
537                                )[1]
538                            ).item()
539                            for i in range(self.n_series)
540                        ]
541                    )
542
543                predicted_cooked_new_X = np.asarray(
544                    [
545                        np.asarray(
546                            self.fit_objs_[i].predict(cooked_new_X)
547                        ).item()
548                        for i in range(self.n_series)
549                    ]
550                )
551
552                preds = np.asarray(y_means_ + predicted_cooked_new_X)
553
554                self.mean_ = mo.rbind(preds, self.mean_)
555
556        # function's return ----------------------------------------------------------------------
557        self.mean_ = pd.DataFrame(
558            self.mean_[0:h, :][::-1],
559            columns=self.df_.columns,
560            index=self.output_dates_,
561        )
562        if "return_std" not in kwargs:
563            if self.kde_ is None:
564                return self.mean_
565
566            # if "return_std" not in kwargs and self.kde_ is not None
567            meanf = []
568            lower = []
569            upper = []
570            self.sims_ = tuple(
571                (
572                    self.mean_ + self.residuals_sims_[i]
573                    for i in tqdm(range(self.replications))
574                )
575            )
576            DescribeResult = namedtuple(
577                "DescribeResult", ("mean", "sims", "lower", "upper")
578            )
579            for ix in range(self.n_series):
580                sims_ix = getsims(self.sims_, ix)
581                if self.agg == "mean":
582                    meanf.append(np.mean(sims_ix, axis=1))
583                else:
584                    meanf.append(np.median(sims_ix, axis=1))
585                lower.append(np.quantile(sims_ix, q=self.alpha_ / 200, axis=1))
586                upper.append(
587                    np.quantile(sims_ix, q=1 - self.alpha_ / 200, axis=1)
588                )
589
590            self.mean_ = pd.DataFrame(
591                np.asarray(meanf).T,
592                columns=self.df_.columns,
593                index=self.output_dates_,
594            )
595
596            self.lower_ = pd.DataFrame(
597                np.asarray(lower).T,
598                columns=self.df_.columns,
599                index=self.output_dates_,
600            )
601
602            self.upper_ = pd.DataFrame(
603                np.asarray(upper).T,
604                columns=self.df_.columns,
605                index=self.output_dates_,
606            )
607
608            return DescribeResult(
609                self.mean_, self.sims_, self.lower_, self.upper_
610            )
611
612        # if "return_std" in kwargs
613        DescribeResult = namedtuple(
614            "DescribeResult", ("mean", "lower", "upper")
615        )
616
617        self.mean_ = pd.DataFrame(
618            np.asarray(self.mean_),
619            columns=self.df_.columns,
620            index=self.output_dates_,
621        )
622
623        self.preds_std_ = np.asarray(self.preds_std_)
624
625        self.lower_ = pd.DataFrame(
626            self.mean_.values - pi_multiplier * self.preds_std_,
627            columns=self.df_.columns,
628            index=self.output_dates_,
629        )
630
631        self.upper_ = pd.DataFrame(
632            self.mean_.values + pi_multiplier * self.preds_std_,
633            columns=self.df_.columns,
634            index=self.output_dates_,
635        )
636
637        return DescribeResult(self.mean_, self.lower_, self.upper_)
638
639    def score(self, X, training_index, testing_index, scoring=None, **kwargs):
640        """Train on training_index, score on testing_index."""
641
642        assert (
643            bool(set(training_index).intersection(set(testing_index))) == False
644        ), "Non-overlapping 'training_index' and 'testing_index' required"
645
646        # Dimensions
647        try:
648            # multivariate time series
649            n, p = X.shape
650        except:
651            # univariate time series
652            n = X.shape[0]
653            p = 1
654
655        # Training and testing sets
656        if p > 1:
657            X_train = X[training_index, :]
658            X_test = X[testing_index, :]
659        else:
660            X_train = X[training_index]
661            X_test = X[testing_index]
662
663        # Horizon
664        h = len(testing_index)
665        assert (
666            len(training_index) + h
667        ) <= n, "Please check lengths of training and testing windows"
668
669        # Fit and predict
670        self.fit(X_train, **kwargs)
671        preds = self.predict(h=h, **kwargs)
672
673        if scoring is None:
674            scoring = "neg_root_mean_squared_error"
675
676        # check inputs
677        assert scoring in (
678            "explained_variance",
679            "neg_mean_absolute_error",
680            "neg_mean_squared_error",
681            "neg_root_mean_squared_error",
682            "neg_mean_squared_log_error",
683            "neg_median_absolute_error",
684            "r2",
685        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
686                               'neg_mean_squared_error', 'neg_root_mean_squared_error', 'neg_mean_squared_log_error', \
687                               'neg_median_absolute_error', 'r2')"
688
689        scoring_options = {
690            "explained_variance": skm2.explained_variance_score,
691            "neg_mean_absolute_error": skm2.mean_absolute_error,
692            "neg_mean_squared_error": skm2.mean_squared_error,
693            "neg_root_mean_squared_error": lambda x, y: np.sqrt(
694                skm2.mean_squared_error(x, y)
695            ),
696            "neg_mean_squared_log_error": skm2.mean_squared_log_error,
697            "neg_median_absolute_error": skm2.median_absolute_error,
698            "r2": skm2.r2_score,
699        }
700
701        # if p > 1:
702        #     return tuple(
703        #         [
704        #             scoring_options[scoring](
705        #                 X_test[:, i], preds[:, i]#, **kwargs
706        #             )
707        #             for i in range(p)
708        #         ]
709        #     )
710        # else:
711        return scoring_options[scoring](X_test, preds)
712
713    def plot(self, series, type_axis="dates", type_plot="pi"):
714        """Plot time series forecast
715
716        Parameters:
717
718            series: {integer} or {string}
719                series index or name
720        """
721
722        assert all(
723            [
724                self.mean_ is not None,
725                self.lower_ is not None,
726                self.upper_ is not None,
727                self.output_dates_ is not None,
728            ]
729        ), "model forecasting must be obtained first (with predict)"
730
731        if isinstance(series, str):
732            assert (
733                series in self.series_names
734            ), f"series {series} doesn't exist in the input dataset"
735            series_idx = self.df_.columns.get_loc(series)
736        else:
737            assert isinstance(series, int) and (
738                0 <= series < self.n_series
739            ), f"check series index (< {self.n_series})"
740            series_idx = series
741
742        y_all = list(self.df_.iloc[:, series_idx]) + list(
743            self.mean_.iloc[:, series_idx]
744        )
745        y_test = list(self.mean_.iloc[:, series_idx])
746        n_points_all = len(y_all)
747        n_points_train = self.df_.shape[0]
748
749        if type_axis == "numeric":
750            x_all = [i for i in range(n_points_all)]
751            x_test = [i for i in range(n_points_train, n_points_all)]
752
753        if type_axis == "dates":  # use dates
754            x_all = np.concatenate(
755                (self.input_dates.values, self.output_dates_.values), axis=None
756            )
757            x_test = self.output_dates_.values
758
759        if type_plot == "pi":
760            fig, ax = plt.subplots()
761            ax.plot(x_all, y_all, "-")
762            ax.plot(x_test, y_test, "-", color="orange")
763            ax.fill_between(
764                x_test,
765                self.lower_.iloc[:, series_idx],
766                self.upper_.iloc[:, series_idx],
767                alpha=0.2,
768                color="orange",
769            )
770            plt.title(
771                f"prediction intervals for {self.replications} simulations of {series}",
772                loc="left",
773                fontsize=12,
774                fontweight=0,
775                color="black",
776            )
777            plt.show()
778
779        if type_plot == "spaghetti":
780            palette = plt.get_cmap("Set1")
781            sims_ix = getsims(self.sims_, series_idx)
782            plt.plot(x_all, y_all, "-")
783            for col_ix in range(
784                sims_ix.shape[1]
785            ):  # avoid this when there are thousands of simulations
786                plt.plot(
787                    x_test,
788                    sims_ix[:, col_ix],
789                    "-",
790                    color=palette(col_ix),
791                    linewidth=1,
792                    alpha=0.9,
793                )
794            plt.plot(x_all, y_all, "-", color="black")
795            plt.plot(x_test, y_test, "-", color="blue")
796            # Add titles
797            plt.title(
798                f"{self.replications} simulations of {series}",
799                loc="left",
800                fontsize=12,
801                fontweight=0,
802                color="black",
803            )
804            plt.xlabel("Time")
805            plt.ylabel("Values")
806            # Show the graph
807            plt.show()

Univariate and multivariate time series (MTS) forecasting with Quasi-Randomized networks (Work in progress /!)

Parameters:

obj: object.
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict()).

n_hidden_features: int.
    number of nodes in the hidden layer.

activation_name: str.
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'.

a: float.
    hyperparameter for 'prelu' or 'elu' activation function.

nodes_sim: str.
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'.

bias: boolean.
    indicates if the hidden layer contains a bias term (True) or not
    (False).

dropout: float.
    regularization parameter; (random) percentage of nodes dropped out
    of the training.

direct_link: boolean.
    indicates if the original predictors are included (True) in model's fitting or not (False).

n_clusters: int.
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering).

cluster_encode: bool.
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding.

type_clust: str.
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm').

type_scaling: a tuple of 3 strings.
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax').

lags: int.
    number of lags used for each time series.

replications: int.
    number of replications (if needed, for predictive simulation). Default is 'None'.

kernel: str.
    the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'.

agg: str.
    either "mean" or "median" for simulation of bootstrap aggregating

seed: int.
    reproducibility seed for nodes_sim=='uniform' or predictive simulation.

backend: str.
    "cpu" or "gpu" or "tpu".

verbose: int.
    0: not printing; 1: printing

show_progress: bool.
    True: progress bar when fitting each series; False: no progress bar when fitting each series

Attributes:

fit_objs_: dict
    objects adjusted to each individual time series

y_: {array-like}
    MTS responses (most recent observations first)

X_: {array-like}
    MTS lags

xreg_: {array-like}
    external regressors

y_means_: dict
    a dictionary of each series mean values

preds_: {array-like}
    successive model predictions

preds_std_: {array-like}
    standard deviation around the predictions

return_std_: boolean
    return uncertainty or not (set in predict)

df_: data frame
    the input data frame, in case a data.frame is provided to `fit`

Examples:

Example 1:

import nnetsauce as ns
import numpy as np
from sklearn import linear_model
np.random.seed(123)

M = np.random.rand(10, 3)
M[:,0] = 10*M[:,0]
M[:,2] = 25*M[:,2]
print(M)

# Adjust Bayesian Ridge
regr4 = linear_model.BayesianRidge()
obj_MTS = ns.MTS(regr4, lags = 1, n_hidden_features=5)
obj_MTS.fit(M)
print(obj_MTS.predict())

# with credible intervals
print(obj_MTS.predict(return_std=True, level=80))

print(obj_MTS.predict(return_std=True, level=95))

Example 2:

import nnetsauce as ns
import numpy as np
from sklearn import linear_model

dataset = {
'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'],
'series1' : [34, 30, 35.6, 33.3, 38.1],
'series2' : [4, 5.5, 5.6, 6.3, 5.1],
'series3' : [100, 100.5, 100.6, 100.2, 100.1]}
df = pd.DataFrame(dataset).set_index('date')
print(df)

# Adjust Bayesian Ridge
regr5 = linear_model.BayesianRidge()
obj_MTS = ns.MTS(regr5, lags = 1, n_hidden_features=5)
obj_MTS.fit(df)
print(obj_MTS.predict())

# with credible intervals
print(obj_MTS.predict(return_std=True, level=80))

print(obj_MTS.predict(return_std=True, level=95))
MTS( obj, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), lags=1, type_pi='kde', replications=None, kernel=None, agg='mean', seed=123, backend='cpu', verbose=0, show_progress=True)
182    def __init__(
183        self,
184        obj,
185        n_hidden_features=5,
186        activation_name="relu",
187        a=0.01,
188        nodes_sim="sobol",
189        bias=True,
190        dropout=0,
191        direct_link=True,
192        n_clusters=2,
193        cluster_encode=True,
194        type_clust="kmeans",
195        type_scaling=("std", "std", "std"),
196        lags=1,
197        type_pi="kde",
198        replications=None,
199        kernel=None,
200        agg="mean",
201        seed=123,
202        backend="cpu",
203        verbose=0,
204        show_progress=True,
205    ):
206        assert int(lags) == lags, "parameter 'lags' should be an integer"
207
208        super().__init__(
209            n_hidden_features=n_hidden_features,
210            activation_name=activation_name,
211            a=a,
212            nodes_sim=nodes_sim,
213            bias=bias,
214            dropout=dropout,
215            direct_link=direct_link,
216            n_clusters=n_clusters,
217            cluster_encode=cluster_encode,
218            type_clust=type_clust,
219            type_scaling=type_scaling,
220            seed=seed,
221            backend=backend,
222        )
223
224        self.obj = obj
225        self.n_series = None
226        self.lags = lags
227        self.type_pi = type_pi
228        self.replications = replications
229        self.kernel = kernel
230        self.agg = agg
231        self.verbose = verbose
232        self.show_progress = show_progress
233        self.series_names = None
234        self.input_dates = None
235        self.fit_objs_ = {}
236        self.y_ = None  # MTS responses (most recent observations first)
237        self.X_ = None  # MTS lags
238        self.xreg_ = None
239        self.y_means_ = {}
240        self.mean_ = None
241        self.upper_ = None
242        self.lower_ = None
243        self.output_dates_ = None
244        self.preds_std_ = []
245        self.alpha_ = None
246        self.return_std_ = None
247        self.df_ = None
248        self.residuals_ = []
249        self.residuals_sims_ = None
250        self.kde_ = None
251        self.sims_ = None
obj
n_series
lags
type_pi
replications
kernel
agg
verbose
show_progress
series_names
input_dates
fit_objs_
y_
X_
xreg_
y_means_
mean_
upper_
lower_
output_dates_
preds_std_
alpha_
return_std_
df_
residuals_
residuals_sims_
kde_
sims_
def fit(self, X, xreg=None, **kwargs):
253    def fit(self, X, xreg=None, **kwargs):
254        """Fit MTS model to training data X, with optional regressors xreg
255
256        Parameters:
257
258            X: {array-like}, shape = [n_samples, n_features]
259                Training time series, where n_samples is the number
260                of samples and n_features is the number of features;
261                X must be in increasing order (most recent observations last)
262
263            xreg: {array-like}, shape = [n_samples, n_features_xreg]
264                Additional regressors to be passed to obj
265                xreg must be in increasing order (most recent observations last)
266
267            **kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity)
268
269        Returns:
270
271            self: object
272        """
273
274        if isinstance(X, pd.DataFrame) is False:
275            self.series_names = ["series" + str(i) for i in range(X.shape[1])]
276            X = pd.DataFrame(X, columns=self.series_names)
277        else:
278            X = copy.deepcopy(mo.convert_df_to_numeric(X))
279            self.series_names = list(X.columns.values)
280
281        self.df_ = X
282        X = X.values
283        self.input_dates = ts.compute_input_dates(self.df_)
284
285        try:
286            # multivariate time series
287            n, p = X.shape
288        except:
289            # univariate time series
290            n = X.shape[0]
291            p = 1
292
293        rep_1_n = np.repeat(1, n)
294
295        self.y_ = None
296        self.X_ = None
297        self.n_series = p
298        self.fit_objs_.clear()
299        self.y_means_.clear()
300        residuals_ = []
301        self.residuals_ = None
302        self.residuals_sims_ = None
303        self.kde_ = None
304        self.sims_ = None
305        self.scaled_Z_ = None
306        self.centered_y_is_ = []
307
308        if p > 1:
309            # multivariate time series
310            mts_input = ts.create_train_inputs(X[::-1], self.lags)
311        else:
312            # univariate time series
313            mts_input = ts.create_train_inputs(
314                X.reshape(-1, 1)[::-1], self.lags
315            )
316
317        self.y_ = mts_input[0]
318
319        self.X_ = mts_input[1]
320
321        if xreg is not None:
322            if isinstance(xreg, pd.DataFrame):
323                xreg = xreg.values
324
325            assert (
326                xreg.shape[0] == n
327            ), "'xreg' and 'X' must have the same number of observations"
328
329            self.xreg_ = xreg
330
331            xreg_input = ts.create_train_inputs(xreg[::-1], self.lags)
332
333            dummy_y, scaled_Z = self.cook_training_set(
334                y=rep_1_n,
335                X=mo.cbind(self.X_, xreg_input[1], backend=self.backend),
336            )
337
338        else:  # xreg is None
339            # avoids scaling X p times in the loop
340            dummy_y, scaled_Z = self.cook_training_set(y=rep_1_n, X=self.X_)
341
342        self.scaled_Z_ = scaled_Z
343
344        # loop on all the time series and adjust self.obj.fit
345        if self.verbose > 0:
346            print(
347                f"\n Adjusting {type(self.obj).__name__} to multivariate time series... \n "
348            )
349
350        if self.show_progress is True:
351            iterator = tqdm(range(p))
352        else:
353            iterator = range(p)
354
355        for i in iterator:
356            y_mean = np.mean(self.y_[:, i])
357            self.y_means_[i] = y_mean
358            centered_y_i = self.y_[:, i] - y_mean
359            self.centered_y_is_.append(centered_y_i)
360            self.obj.fit(X=scaled_Z, y=centered_y_i)
361            self.fit_objs_[i] = deepcopy(self.obj)
362            residuals_.append(
363                (centered_y_i - self.fit_objs_[i].predict(scaled_Z)).tolist()
364            )
365
366        self.residuals_ = np.asarray(residuals_).T
367
368        if self.replications != None and self.type_pi == "kde":
369            if self.verbose > 0:
370                print(f"\n Simulate residuals using {self.kernel} kernel... \n")
371            assert self.kernel in (
372                "gaussian",
373                "tophat",
374            ), "currently, 'kernel' must be either 'gaussian' or 'tophat'"
375            kernel_bandwidths = {"bandwidth": np.logspace(-6, 6, 150)}
376            grid = GridSearchCV(
377                KernelDensity(kernel=self.kernel, **kwargs),
378                param_grid=kernel_bandwidths,
379            )
380            grid.fit(self.residuals_)
381
382            if self.verbose > 0:
383                print(
384                    f"\n Best parameters for {self.kernel} kernel: {grid.best_params_} \n"
385                )
386
387            self.kde_ = grid.best_estimator_
388
389        return self

Fit MTS model to training data X, with optional regressors xreg

Parameters:

X: {array-like}, shape = [n_samples, n_features]
    Training time series, where n_samples is the number
    of samples and n_features is the number of features;
    X must be in increasing order (most recent observations last)

xreg: {array-like}, shape = [n_samples, n_features_xreg]
    Additional regressors to be passed to obj
    xreg must be in increasing order (most recent observations last)

**kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity)

Returns:

self: object
def predict(self, h=5, level=95, new_xreg=None, **kwargs):
391    def predict(self, h=5, level=95, new_xreg=None, **kwargs):
392        """Forecast all the time series, h steps ahead
393
394        Parameters:
395
396            h: {integer}
397                Forecasting horizon
398
399            level: {integer}
400                Level of confidence (if obj has option 'return_std' and the
401                posterior is gaussian)
402
403            new_xreg: {array-like}, shape = [n_samples = h, n_new_xreg]
404                New values of additional (deterministic) regressors on horizon = h
405                new_xreg must be in increasing order (most recent observations last)
406
407            **kwargs: additional parameters to be passed to
408                    self.cook_test_set
409
410        Returns:
411
412            model predictions for horizon = h: {array-like}, data frame or tuple.
413            Standard deviation and prediction intervals are returned when
414            `obj.predict` can return standard deviation
415        """
416
417        self.output_dates_, frequency = ts.compute_output_dates(self.df_, h)
418
419        self.return_std_ = False  # do not remove (/!\)
420
421        self.mean_ = None  # do not remove (/!\)
422
423        self.mean_ = deepcopy(self.y_)  # do not remove (/!\)
424
425        self.lower_ = None
426
427        self.upper_ = None
428
429        self.sims_ = None
430
431        y_means_ = np.asarray([self.y_means_[i] for i in range(self.n_series)])
432
433        n_features = self.n_series * self.lags
434
435        self.alpha_ = 100 - level
436
437        if "return_std" in kwargs:
438            self.return_std_ = True
439            self.preds_std_ = []
440            pi_multiplier = norm.ppf(1 - self.alpha_ / 200)
441            DescribeResult = namedtuple(
442                "DescribeResult", ("mean", "lower", "upper")
443            )  # to be updated
444
445        if self.xreg_ is None:  # no external regressors
446            if self.kde_ != None and self.type_pi == "kde":
447                self.residuals_sims_ = tuple(
448                    self.kde_.sample(
449                        n_samples=h, random_state=self.seed + 100 * i
450                    )
451                    for i in tqdm(range(self.replications))
452                )
453
454            for _ in range(h):
455                new_obs = ts.reformat_response(self.mean_, self.lags)
456
457                new_X = new_obs.reshape(1, n_features)
458
459                cooked_new_X = self.cook_test_set(new_X, **kwargs)
460
461                if "return_std" in kwargs:
462                    self.preds_std_.append(
463                        [
464                            np.asarray(
465                                self.fit_objs_[i].predict(
466                                    cooked_new_X, return_std=True
467                                )[1]
468                            ).item()
469                            for i in range(self.n_series)
470                        ]
471                    )
472
473                predicted_cooked_new_X = np.asarray(
474                    [
475                        np.asarray(
476                            self.fit_objs_[i].predict(cooked_new_X)
477                        ).item()
478                        for i in range(self.n_series)
479                    ]
480                )
481
482                preds = np.asarray(y_means_ + predicted_cooked_new_X)
483
484                self.mean_ = mo.rbind(preds, self.mean_)
485
486        else:  # if self.xreg_ is not None: # with external regressors
487            assert (
488                new_xreg is not None
489            ), "'new_xreg' must be provided to predict()"
490
491            if isinstance(new_xreg, pd.DataFrame):
492                new_xreg = new_xreg.values
493
494            if self.kde_ is not None:
495                if self.verbose > 0:
496                    print("\n Obtain simulations for adjusted residuals... \n")
497                self.residuals_sims_ = tuple(
498                    self.kde_.sample(
499                        n_samples=h, random_state=self.seed + 100 * i
500                    )
501                    for i in tqdm(range(self.replications))
502                )
503
504            try:
505                n_obs_xreg, n_features_xreg = new_xreg.shape
506                assert (
507                    n_features_xreg == self.xreg_.shape[1]
508                ), "check number of inputs provided for 'new_xreg' (compare with self.xreg_.shape[1])"
509            except:
510                n_obs_xreg = new_xreg.shape  # one series
511
512            assert (
513                n_obs_xreg == h
514            ), "please provide values of regressors 'new_xreg' for the whole horizon 'h'"
515
516            n_features_xreg = n_features_xreg * self.lags
517
518            inv_new_xreg = mo.rbind(self.xreg_, new_xreg)[::-1]
519
520            for _ in range(h):
521                new_obs = ts.reformat_response(self.mean_, self.lags)
522
523                new_obs_xreg = ts.reformat_response(inv_new_xreg, self.lags)
524
525                new_X = np.concatenate(
526                    (new_obs, new_obs_xreg), axis=None
527                ).reshape(1, -1)
528
529                cooked_new_X = self.cook_test_set(new_X, **kwargs)
530
531                if "return_std" in kwargs:
532                    self.preds_std_.append(
533                        [
534                            np.asarray(
535                                self.fit_objs_[i].predict(
536                                    cooked_new_X, return_std=True
537                                )[1]
538                            ).item()
539                            for i in range(self.n_series)
540                        ]
541                    )
542
543                predicted_cooked_new_X = np.asarray(
544                    [
545                        np.asarray(
546                            self.fit_objs_[i].predict(cooked_new_X)
547                        ).item()
548                        for i in range(self.n_series)
549                    ]
550                )
551
552                preds = np.asarray(y_means_ + predicted_cooked_new_X)
553
554                self.mean_ = mo.rbind(preds, self.mean_)
555
556        # function's return ----------------------------------------------------------------------
557        self.mean_ = pd.DataFrame(
558            self.mean_[0:h, :][::-1],
559            columns=self.df_.columns,
560            index=self.output_dates_,
561        )
562        if "return_std" not in kwargs:
563            if self.kde_ is None:
564                return self.mean_
565
566            # if "return_std" not in kwargs and self.kde_ is not None
567            meanf = []
568            lower = []
569            upper = []
570            self.sims_ = tuple(
571                (
572                    self.mean_ + self.residuals_sims_[i]
573                    for i in tqdm(range(self.replications))
574                )
575            )
576            DescribeResult = namedtuple(
577                "DescribeResult", ("mean", "sims", "lower", "upper")
578            )
579            for ix in range(self.n_series):
580                sims_ix = getsims(self.sims_, ix)
581                if self.agg == "mean":
582                    meanf.append(np.mean(sims_ix, axis=1))
583                else:
584                    meanf.append(np.median(sims_ix, axis=1))
585                lower.append(np.quantile(sims_ix, q=self.alpha_ / 200, axis=1))
586                upper.append(
587                    np.quantile(sims_ix, q=1 - self.alpha_ / 200, axis=1)
588                )
589
590            self.mean_ = pd.DataFrame(
591                np.asarray(meanf).T,
592                columns=self.df_.columns,
593                index=self.output_dates_,
594            )
595
596            self.lower_ = pd.DataFrame(
597                np.asarray(lower).T,
598                columns=self.df_.columns,
599                index=self.output_dates_,
600            )
601
602            self.upper_ = pd.DataFrame(
603                np.asarray(upper).T,
604                columns=self.df_.columns,
605                index=self.output_dates_,
606            )
607
608            return DescribeResult(
609                self.mean_, self.sims_, self.lower_, self.upper_
610            )
611
612        # if "return_std" in kwargs
613        DescribeResult = namedtuple(
614            "DescribeResult", ("mean", "lower", "upper")
615        )
616
617        self.mean_ = pd.DataFrame(
618            np.asarray(self.mean_),
619            columns=self.df_.columns,
620            index=self.output_dates_,
621        )
622
623        self.preds_std_ = np.asarray(self.preds_std_)
624
625        self.lower_ = pd.DataFrame(
626            self.mean_.values - pi_multiplier * self.preds_std_,
627            columns=self.df_.columns,
628            index=self.output_dates_,
629        )
630
631        self.upper_ = pd.DataFrame(
632            self.mean_.values + pi_multiplier * self.preds_std_,
633            columns=self.df_.columns,
634            index=self.output_dates_,
635        )
636
637        return DescribeResult(self.mean_, self.lower_, self.upper_)

Forecast all the time series, h steps ahead

Parameters:

h: {integer}
    Forecasting horizon

level: {integer}
    Level of confidence (if obj has option 'return_std' and the
    posterior is gaussian)

new_xreg: {array-like}, shape = [n_samples = h, n_new_xreg]
    New values of additional (deterministic) regressors on horizon = h
    new_xreg must be in increasing order (most recent observations last)

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions for horizon = h: {array-like}, data frame or tuple.
Standard deviation and prediction intervals are returned when
`obj.predict` can return standard deviation
def score(self, X, training_index, testing_index, scoring=None, **kwargs):
639    def score(self, X, training_index, testing_index, scoring=None, **kwargs):
640        """Train on training_index, score on testing_index."""
641
642        assert (
643            bool(set(training_index).intersection(set(testing_index))) == False
644        ), "Non-overlapping 'training_index' and 'testing_index' required"
645
646        # Dimensions
647        try:
648            # multivariate time series
649            n, p = X.shape
650        except:
651            # univariate time series
652            n = X.shape[0]
653            p = 1
654
655        # Training and testing sets
656        if p > 1:
657            X_train = X[training_index, :]
658            X_test = X[testing_index, :]
659        else:
660            X_train = X[training_index]
661            X_test = X[testing_index]
662
663        # Horizon
664        h = len(testing_index)
665        assert (
666            len(training_index) + h
667        ) <= n, "Please check lengths of training and testing windows"
668
669        # Fit and predict
670        self.fit(X_train, **kwargs)
671        preds = self.predict(h=h, **kwargs)
672
673        if scoring is None:
674            scoring = "neg_root_mean_squared_error"
675
676        # check inputs
677        assert scoring in (
678            "explained_variance",
679            "neg_mean_absolute_error",
680            "neg_mean_squared_error",
681            "neg_root_mean_squared_error",
682            "neg_mean_squared_log_error",
683            "neg_median_absolute_error",
684            "r2",
685        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
686                               'neg_mean_squared_error', 'neg_root_mean_squared_error', 'neg_mean_squared_log_error', \
687                               'neg_median_absolute_error', 'r2')"
688
689        scoring_options = {
690            "explained_variance": skm2.explained_variance_score,
691            "neg_mean_absolute_error": skm2.mean_absolute_error,
692            "neg_mean_squared_error": skm2.mean_squared_error,
693            "neg_root_mean_squared_error": lambda x, y: np.sqrt(
694                skm2.mean_squared_error(x, y)
695            ),
696            "neg_mean_squared_log_error": skm2.mean_squared_log_error,
697            "neg_median_absolute_error": skm2.median_absolute_error,
698            "r2": skm2.r2_score,
699        }
700
701        # if p > 1:
702        #     return tuple(
703        #         [
704        #             scoring_options[scoring](
705        #                 X_test[:, i], preds[:, i]#, **kwargs
706        #             )
707        #             for i in range(p)
708        #         ]
709        #     )
710        # else:
711        return scoring_options[scoring](X_test, preds)

Train on training_index, score on testing_index.

def plot(self, series, type_axis='dates', type_plot='pi'):
713    def plot(self, series, type_axis="dates", type_plot="pi"):
714        """Plot time series forecast
715
716        Parameters:
717
718            series: {integer} or {string}
719                series index or name
720        """
721
722        assert all(
723            [
724                self.mean_ is not None,
725                self.lower_ is not None,
726                self.upper_ is not None,
727                self.output_dates_ is not None,
728            ]
729        ), "model forecasting must be obtained first (with predict)"
730
731        if isinstance(series, str):
732            assert (
733                series in self.series_names
734            ), f"series {series} doesn't exist in the input dataset"
735            series_idx = self.df_.columns.get_loc(series)
736        else:
737            assert isinstance(series, int) and (
738                0 <= series < self.n_series
739            ), f"check series index (< {self.n_series})"
740            series_idx = series
741
742        y_all = list(self.df_.iloc[:, series_idx]) + list(
743            self.mean_.iloc[:, series_idx]
744        )
745        y_test = list(self.mean_.iloc[:, series_idx])
746        n_points_all = len(y_all)
747        n_points_train = self.df_.shape[0]
748
749        if type_axis == "numeric":
750            x_all = [i for i in range(n_points_all)]
751            x_test = [i for i in range(n_points_train, n_points_all)]
752
753        if type_axis == "dates":  # use dates
754            x_all = np.concatenate(
755                (self.input_dates.values, self.output_dates_.values), axis=None
756            )
757            x_test = self.output_dates_.values
758
759        if type_plot == "pi":
760            fig, ax = plt.subplots()
761            ax.plot(x_all, y_all, "-")
762            ax.plot(x_test, y_test, "-", color="orange")
763            ax.fill_between(
764                x_test,
765                self.lower_.iloc[:, series_idx],
766                self.upper_.iloc[:, series_idx],
767                alpha=0.2,
768                color="orange",
769            )
770            plt.title(
771                f"prediction intervals for {self.replications} simulations of {series}",
772                loc="left",
773                fontsize=12,
774                fontweight=0,
775                color="black",
776            )
777            plt.show()
778
779        if type_plot == "spaghetti":
780            palette = plt.get_cmap("Set1")
781            sims_ix = getsims(self.sims_, series_idx)
782            plt.plot(x_all, y_all, "-")
783            for col_ix in range(
784                sims_ix.shape[1]
785            ):  # avoid this when there are thousands of simulations
786                plt.plot(
787                    x_test,
788                    sims_ix[:, col_ix],
789                    "-",
790                    color=palette(col_ix),
791                    linewidth=1,
792                    alpha=0.9,
793                )
794            plt.plot(x_all, y_all, "-", color="black")
795            plt.plot(x_test, y_test, "-", color="blue")
796            # Add titles
797            plt.title(
798                f"{self.replications} simulations of {series}",
799                loc="left",
800                fontsize=12,
801                fontweight=0,
802                color="black",
803            )
804            plt.xlabel("Time")
805            plt.ylabel("Values")
806            # Show the graph
807            plt.show()

Plot time series forecast

Parameters:

series: {integer} or {string}
    series index or name
def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_predict_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class MultitaskClassifier(nnetsauce.Base, sklearn.base.ClassifierMixin):
 16class MultitaskClassifier(Base, ClassifierMixin):
 17    """Multitask Classification model based on regression models, with shared covariates
 18
 19    Parameters:
 20
 21        obj: object
 22            any object (must be a regression model) containing a method fit (obj.fit())
 23            and a method predict (obj.predict())
 24
 25        n_hidden_features: int
 26            number of nodes in the hidden layer
 27
 28        activation_name: str
 29            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 30
 31        a: float
 32            hyperparameter for 'prelu' or 'elu' activation function
 33
 34        nodes_sim: str
 35            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 36            'uniform'
 37
 38        bias: boolean
 39            indicates if the hidden layer contains a bias term (True) or not
 40            (False)
 41
 42        dropout: float
 43            regularization parameter; (random) percentage of nodes dropped out
 44            of the training
 45
 46        direct_link: boolean
 47            indicates if the original predictors are included (True) in model's
 48            fitting or not (False)
 49
 50        n_clusters: int
 51            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 52                no clustering)
 53
 54        cluster_encode: bool
 55            defines how the variable containing clusters is treated (default is one-hot)
 56            if `False`, then labels are used, without one-hot encoding
 57
 58        type_clust: str
 59            type of clustering method: currently k-means ('kmeans') or Gaussian
 60            Mixture Model ('gmm')
 61
 62        type_scaling: a tuple of 3 strings
 63            scaling methods for inputs, hidden layer, and clustering respectively
 64            (and when relevant).
 65            Currently available: standardization ('std') or MinMax scaling ('minmax')
 66
 67        col_sample: float
 68            percentage of covariates randomly chosen for training
 69
 70        row_sample: float
 71            percentage of rows chosen for training, by stratified bootstrapping
 72
 73        seed: int
 74            reproducibility seed for nodes_sim=='uniform'
 75
 76        backend: str
 77            "cpu" or "gpu" or "tpu"
 78
 79    Attributes:
 80
 81        fit_objs_: dict
 82            objects adjusted to each individual time series
 83
 84        n_classes_: int
 85            number of classes for the classifier
 86
 87    Examples:
 88
 89    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/mtask_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/mtask_classification.py)
 90
 91    ```python
 92    import nnetsauce as ns
 93    import numpy as np
 94    from sklearn.datasets import load_breast_cancer
 95    from sklearn.linear_model import LinearRegression
 96    from sklearn.model_selection import train_test_split
 97    from sklearn import metrics
 98    from time import time
 99
100    breast_cancer = load_breast_cancer()
101    Z = breast_cancer.data
102    t = breast_cancer.target
103
104    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2,
105                                                        random_state=123+2*10)
106
107    # Linear Regression is used
108    regr = LinearRegression()
109    fit_obj = ns.MultitaskClassifier(regr, n_hidden_features=5,
110                                n_clusters=2, type_clust="gmm")
111
112    start = time()
113    fit_obj.fit(X_train, y_train)
114    print(f"Elapsed {time() - start}")
115
116    print(fit_obj.score(X_test, y_test))
117    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
118
119    start = time()
120    preds = fit_obj.predict(X_test)
121    print(f"Elapsed {time() - start}")
122    print(metrics.classification_report(preds, y_test))
123    ```
124
125    """
126
127    # construct the object -----
128
129    def __init__(
130        self,
131        obj,
132        n_hidden_features=5,
133        activation_name="relu",
134        a=0.01,
135        nodes_sim="sobol",
136        bias=True,
137        dropout=0,
138        direct_link=True,
139        n_clusters=2,
140        cluster_encode=True,
141        type_clust="kmeans",
142        type_scaling=("std", "std", "std"),
143        col_sample=1,
144        row_sample=1,
145        seed=123,
146        backend="cpu",
147    ):
148        super().__init__(
149            n_hidden_features=n_hidden_features,
150            activation_name=activation_name,
151            a=a,
152            nodes_sim=nodes_sim,
153            bias=bias,
154            dropout=dropout,
155            direct_link=direct_link,
156            n_clusters=n_clusters,
157            cluster_encode=cluster_encode,
158            type_clust=type_clust,
159            type_scaling=type_scaling,
160            col_sample=col_sample,
161            row_sample=row_sample,
162            seed=seed,
163            backend=backend,
164        )
165
166        self.type_fit = "classification"
167        self.obj = obj
168        self.fit_objs_ = {}
169
170    def fit(self, X, y, sample_weight=None, **kwargs):
171        """Fit MultitaskClassifier to training data (X, y).
172
173        Args:
174
175            X: {array-like}, shape = [n_samples, n_features]
176                Training vectors, where n_samples is the number
177                of samples and n_features is the number of features.
178
179            y: array-like, shape = [n_samples]
180                Target values.
181
182            **kwargs: additional parameters to be passed to
183                    self.cook_training_set or self.obj.fit
184
185        Returns:
186
187            self: object
188
189        """
190
191        assert mx.is_factor(y), "y must contain only integers"
192
193        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
194
195        self.n_classes_ = len(np.unique(y))
196
197        # multitask response
198        Y = mo.one_hot_encode2(output_y, self.n_classes_)
199
200        # if sample_weight is None:
201        for i in range(self.n_classes_):
202            self.fit_objs_[i] = pickle.loads(
203                pickle.dumps(self.obj.fit(scaled_Z, Y[:, i], **kwargs), -1)
204            )
205
206        return self
207
208    def predict(self, X, **kwargs):
209        """Predict test data X.
210
211        Args:
212
213            X: {array-like}, shape = [n_samples, n_features]
214                Training vectors, where n_samples is the number
215                of samples and n_features is the number of features.
216
217            **kwargs: additional parameters to be passed to
218                    self.cook_test_set
219
220        Returns:
221
222            model predictions: {array-like}
223
224        """
225
226        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
227
228    def predict_proba(self, X, **kwargs):
229        """Predict probabilities for test data X.
230
231        Args:
232
233            X: {array-like}, shape = [n_samples, n_features]
234                Training vectors, where n_samples is the number
235                of samples and n_features is the number of features.
236
237            **kwargs: additional parameters to be passed to
238                    self.cook_test_set
239
240        Returns:
241
242            probability estimates for test data: {array-like}
243
244        """
245
246        shape_X = X.shape
247
248        probs = np.zeros((shape_X[0], self.n_classes_))
249
250        if len(shape_X) == 1:
251            n_features = shape_X[0]
252
253            new_X = mo.rbind(
254                X.reshape(1, n_features),
255                np.ones(n_features).reshape(1, n_features),
256            )
257
258            Z = self.cook_test_set(new_X, **kwargs)
259
260            # loop on all the classes
261            for i in range(self.n_classes_):
262                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0]
263
264        else:
265            Z = self.cook_test_set(X, **kwargs)
266
267            # loop on all the classes
268            for i in range(self.n_classes_):
269                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)
270
271        expit_raw_probs = expit(probs)
272
273        return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None]
274
275    def score(self, X, y, scoring=None, **kwargs):
276        """ Score the model on test set features X and response y. 
277
278        Args:
279        
280            X: {array-like}, shape = [n_samples, n_features]
281                Training vectors, where n_samples is the number 
282                of samples and n_features is the number of features
283
284            y: array-like, shape = [n_samples]
285                Target values
286
287            scoring: str
288                must be in ('explained_variance', 'neg_mean_absolute_error', \
289                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
290                            'neg_median_absolute_error', 'r2')
291            
292            **kwargs: additional parameters to be passed to scoring functions
293               
294        Returns: 
295        
296            model scores: {array-like}
297
298        """
299
300        preds = self.predict(X)
301
302        if scoring is None:
303            scoring = "accuracy"
304
305        # check inputs
306        assert scoring in (
307            "accuracy",
308            "average_precision",
309            "brier_score_loss",
310            "f1",
311            "f1_micro",
312            "f1_macro",
313            "f1_weighted",
314            "f1_samples",
315            "neg_log_loss",
316            "precision",
317            "recall",
318            "roc_auc",
319        ), "'scoring' should be in ('accuracy', 'average_precision', \
320                           'brier_score_loss', 'f1', 'f1_micro', \
321                           'f1_macro', 'f1_weighted',  'f1_samples', \
322                           'neg_log_loss', 'precision', 'recall', \
323                           'roc_auc')"
324
325        scoring_options = {
326            "accuracy": skm2.accuracy_score,
327            "average_precision": skm2.average_precision_score,
328            "brier_score_loss": skm2.brier_score_loss,
329            "f1": skm2.f1_score,
330            "f1_micro": skm2.f1_score,
331            "f1_macro": skm2.f1_score,
332            "f1_weighted": skm2.f1_score,
333            "f1_samples": skm2.f1_score,
334            "neg_log_loss": skm2.log_loss,
335            "precision": skm2.precision_score,
336            "recall": skm2.recall_score,
337            "roc_auc": skm2.roc_auc_score,
338        }
339
340        return scoring_options[scoring](y, preds, **kwargs)

Multitask Classification model based on regression models, with shared covariates

Parameters:

obj: object
    any object (must be a regression model) containing a method fit (obj.fit())
    and a method predict (obj.predict())

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

fit_objs_: dict
    objects adjusted to each individual time series

n_classes_: int
    number of classes for the classifier

Examples:

See also https://github.com/Techtonique/nnetsauce/blob/master/examples/mtask_classification.py

import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time

breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target

X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2,
                                                    random_state=123+2*10)

# Linear Regression is used
regr = LinearRegression()
fit_obj = ns.MultitaskClassifier(regr, n_hidden_features=5,
                            n_clusters=2, type_clust="gmm")

start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))

start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
MultitaskClassifier( obj, n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, seed=123, backend='cpu')
129    def __init__(
130        self,
131        obj,
132        n_hidden_features=5,
133        activation_name="relu",
134        a=0.01,
135        nodes_sim="sobol",
136        bias=True,
137        dropout=0,
138        direct_link=True,
139        n_clusters=2,
140        cluster_encode=True,
141        type_clust="kmeans",
142        type_scaling=("std", "std", "std"),
143        col_sample=1,
144        row_sample=1,
145        seed=123,
146        backend="cpu",
147    ):
148        super().__init__(
149            n_hidden_features=n_hidden_features,
150            activation_name=activation_name,
151            a=a,
152            nodes_sim=nodes_sim,
153            bias=bias,
154            dropout=dropout,
155            direct_link=direct_link,
156            n_clusters=n_clusters,
157            cluster_encode=cluster_encode,
158            type_clust=type_clust,
159            type_scaling=type_scaling,
160            col_sample=col_sample,
161            row_sample=row_sample,
162            seed=seed,
163            backend=backend,
164        )
165
166        self.type_fit = "classification"
167        self.obj = obj
168        self.fit_objs_ = {}
type_fit
obj
fit_objs_
def fit(self, X, y, sample_weight=None, **kwargs):
170    def fit(self, X, y, sample_weight=None, **kwargs):
171        """Fit MultitaskClassifier to training data (X, y).
172
173        Args:
174
175            X: {array-like}, shape = [n_samples, n_features]
176                Training vectors, where n_samples is the number
177                of samples and n_features is the number of features.
178
179            y: array-like, shape = [n_samples]
180                Target values.
181
182            **kwargs: additional parameters to be passed to
183                    self.cook_training_set or self.obj.fit
184
185        Returns:
186
187            self: object
188
189        """
190
191        assert mx.is_factor(y), "y must contain only integers"
192
193        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
194
195        self.n_classes_ = len(np.unique(y))
196
197        # multitask response
198        Y = mo.one_hot_encode2(output_y, self.n_classes_)
199
200        # if sample_weight is None:
201        for i in range(self.n_classes_):
202            self.fit_objs_[i] = pickle.loads(
203                pickle.dumps(self.obj.fit(scaled_Z, Y[:, i], **kwargs), -1)
204            )
205
206        return self

Fit MultitaskClassifier to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
208    def predict(self, X, **kwargs):
209        """Predict test data X.
210
211        Args:
212
213            X: {array-like}, shape = [n_samples, n_features]
214                Training vectors, where n_samples is the number
215                of samples and n_features is the number of features.
216
217            **kwargs: additional parameters to be passed to
218                    self.cook_test_set
219
220        Returns:
221
222            model predictions: {array-like}
223
224        """
225
226        return np.argmax(self.predict_proba(X, **kwargs), axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
228    def predict_proba(self, X, **kwargs):
229        """Predict probabilities for test data X.
230
231        Args:
232
233            X: {array-like}, shape = [n_samples, n_features]
234                Training vectors, where n_samples is the number
235                of samples and n_features is the number of features.
236
237            **kwargs: additional parameters to be passed to
238                    self.cook_test_set
239
240        Returns:
241
242            probability estimates for test data: {array-like}
243
244        """
245
246        shape_X = X.shape
247
248        probs = np.zeros((shape_X[0], self.n_classes_))
249
250        if len(shape_X) == 1:
251            n_features = shape_X[0]
252
253            new_X = mo.rbind(
254                X.reshape(1, n_features),
255                np.ones(n_features).reshape(1, n_features),
256            )
257
258            Z = self.cook_test_set(new_X, **kwargs)
259
260            # loop on all the classes
261            for i in range(self.n_classes_):
262                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0]
263
264        else:
265            Z = self.cook_test_set(X, **kwargs)
266
267            # loop on all the classes
268            for i in range(self.n_classes_):
269                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)
270
271        expit_raw_probs = expit(probs)
272
273        return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None]

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
def score(self, X, y, scoring=None, **kwargs):
275    def score(self, X, y, scoring=None, **kwargs):
276        """ Score the model on test set features X and response y. 
277
278        Args:
279        
280            X: {array-like}, shape = [n_samples, n_features]
281                Training vectors, where n_samples is the number 
282                of samples and n_features is the number of features
283
284            y: array-like, shape = [n_samples]
285                Target values
286
287            scoring: str
288                must be in ('explained_variance', 'neg_mean_absolute_error', \
289                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
290                            'neg_median_absolute_error', 'r2')
291            
292            **kwargs: additional parameters to be passed to scoring functions
293               
294        Returns: 
295        
296            model scores: {array-like}
297
298        """
299
300        preds = self.predict(X)
301
302        if scoring is None:
303            scoring = "accuracy"
304
305        # check inputs
306        assert scoring in (
307            "accuracy",
308            "average_precision",
309            "brier_score_loss",
310            "f1",
311            "f1_micro",
312            "f1_macro",
313            "f1_weighted",
314            "f1_samples",
315            "neg_log_loss",
316            "precision",
317            "recall",
318            "roc_auc",
319        ), "'scoring' should be in ('accuracy', 'average_precision', \
320                           'brier_score_loss', 'f1', 'f1_micro', \
321                           'f1_macro', 'f1_weighted',  'f1_samples', \
322                           'neg_log_loss', 'precision', 'recall', \
323                           'roc_auc')"
324
325        scoring_options = {
326            "accuracy": skm2.accuracy_score,
327            "average_precision": skm2.average_precision_score,
328            "brier_score_loss": skm2.brier_score_loss,
329            "f1": skm2.f1_score,
330            "f1_micro": skm2.f1_score,
331            "f1_macro": skm2.f1_score,
332            "f1_weighted": skm2.f1_score,
333            "f1_samples": skm2.f1_score,
334            "neg_log_loss": skm2.log_loss,
335            "precision": skm2.precision_score,
336            "recall": skm2.recall_score,
337            "roc_auc": skm2.roc_auc_score,
338        }
339
340        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                             'neg_mean_squared_error', 'neg_mean_squared_log_error',                             'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class SimpleMultitaskClassifier(nnetsauce.Base, sklearn.base.ClassifierMixin):
 17class SimpleMultitaskClassifier(Base, ClassifierMixin):
 18    """Multitask Classification model based on regression models, with shared covariates
 19
 20    Parameters:
 21
 22        obj: object
 23            any object (must be a regression model) containing a method fit (obj.fit())
 24            and a method predict (obj.predict())
 25
 26        seed: int
 27            reproducibility seed
 28
 29    Attributes:
 30
 31        fit_objs_: dict
 32            objects adjusted to each individual time series
 33
 34        n_classes_: int
 35            number of classes for the classifier
 36
 37    Examples:
 38
 39    ```python
 40    import nnetsauce as ns
 41    import numpy as np
 42    from sklearn.datasets import load_breast_cancer
 43    from sklearn.linear_model import LinearRegression
 44    from sklearn.model_selection import train_test_split
 45    from sklearn import metrics
 46    from time import time
 47
 48    breast_cancer = load_breast_cancer()
 49    Z = breast_cancer.data
 50    t = breast_cancer.target
 51
 52    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2,
 53                                                        random_state=123+2*10)
 54
 55    # Linear Regression is used
 56    regr = LinearRegression()
 57    fit_obj = ns.SimpleMultitaskClassifier(regr)
 58
 59    start = time()
 60    fit_obj.fit(X_train, y_train)
 61    print(f"Elapsed {time() - start}")
 62
 63    print(fit_obj.score(X_test, y_test))
 64    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
 65
 66    start = time()
 67    preds = fit_obj.predict(X_test)
 68    print(f"Elapsed {time() - start}")
 69    print(metrics.classification_report(preds, y_test))
 70    ```
 71
 72    """
 73
 74    # construct the object -----
 75
 76    def __init__(
 77        self,
 78        obj,
 79    ):
 80        self.type_fit = "classification"
 81        self.obj = obj
 82        self.fit_objs_ = {}
 83        self.X_scaler_ = StandardScaler()
 84        self.scaled_X_ = None
 85
 86    def fit(self, X, y, sample_weight=None, **kwargs):
 87        """Fit SimpleMultitaskClassifier to training data (X, y).
 88
 89        Args:
 90
 91            X: {array-like}, shape = [n_samples, n_features]
 92                Training vectors, where n_samples is the number
 93                of samples and n_features is the number of features.
 94
 95            y: array-like, shape = [n_samples]
 96                Target values.
 97
 98            **kwargs: additional parameters to be passed to
 99                    self.cook_training_set or self.obj.fit
100
101        Returns:
102
103            self: object
104
105        """
106
107        assert mx.is_factor(y), "y must contain only integers"
108
109        self.scaled_X_ = self.X_scaler_.fit_transform(X)
110
111        self.n_classes_ = len(np.unique(y))
112
113        # multitask response
114        Y = mo.one_hot_encode2(y, self.n_classes_)
115
116        # if sample_weight is None:
117        for i in range(self.n_classes_):
118            self.fit_objs_[i] = deepcopy(
119                self.obj.fit(self.scaled_X_, Y[:, i], **kwargs)
120            )
121
122        return self
123
124    def predict(self, X, **kwargs):
125        """Predict test data X.
126
127        Args:
128
129            X: {array-like}, shape = [n_samples, n_features]
130                Training vectors, where n_samples is the number
131                of samples and n_features is the number of features.
132
133            **kwargs: additional parameters
134
135        Returns:
136
137            model predictions: {array-like}
138
139        """
140
141        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
142
143    def predict_proba(self, X, **kwargs):
144        """Predict probabilities for test data X.
145
146        Args:
147
148            X: {array-like}, shape = [n_samples, n_features]
149                Training vectors, where n_samples is the number
150                of samples and n_features is the number of features.
151
152            **kwargs: additional parameters
153
154        Returns:
155
156            probability estimates for test data: {array-like}
157
158        """
159
160        shape_X = X.shape
161
162        probs = np.zeros((shape_X[0], self.n_classes_))
163
164        if len(shape_X) == 1:
165            n_features = shape_X[0]
166
167            new_X = mo.rbind(
168                X.reshape(1, n_features),
169                np.ones(n_features).reshape(1, n_features),
170            )
171
172            Z = self.X_scaler_.transform(new_X, **kwargs)
173
174            # loop on all the classes
175            for i in range(self.n_classes_):
176                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0]
177
178        else:
179            Z = self.X_scaler_.transform(X, **kwargs)
180
181            # loop on all the classes
182            for i in range(self.n_classes_):
183                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)
184
185        expit_raw_probs = expit(probs)
186
187        return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None]
188
189    def score(self, X, y, scoring=None, **kwargs):
190        """ Score the model on test set features X and response y. 
191
192        Args:
193        
194            X: {array-like}, shape = [n_samples, n_features]
195                Training vectors, where n_samples is the number 
196                of samples and n_features is the number of features
197
198            y: array-like, shape = [n_samples]
199                Target values
200
201            scoring: str
202                must be in ('explained_variance', 'neg_mean_absolute_error', \
203                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
204                            'neg_median_absolute_error', 'r2')
205            
206            **kwargs: additional parameters to be passed to scoring functions
207               
208        Returns: 
209        
210            model scores: {array-like}
211
212        """
213
214        preds = self.predict(X)
215
216        if scoring is None:
217            scoring = "accuracy"
218
219        # check inputs
220        assert scoring in (
221            "accuracy",
222            "average_precision",
223            "brier_score_loss",
224            "f1",
225            "f1_micro",
226            "f1_macro",
227            "f1_weighted",
228            "f1_samples",
229            "neg_log_loss",
230            "precision",
231            "recall",
232            "roc_auc",
233        ), "'scoring' should be in ('accuracy', 'average_precision', \
234                           'brier_score_loss', 'f1', 'f1_micro', \
235                           'f1_macro', 'f1_weighted',  'f1_samples', \
236                           'neg_log_loss', 'precision', 'recall', \
237                           'roc_auc')"
238
239        scoring_options = {
240            "accuracy": skm2.accuracy_score,
241            "average_precision": skm2.average_precision_score,
242            "brier_score_loss": skm2.brier_score_loss,
243            "f1": skm2.f1_score,
244            "f1_micro": skm2.f1_score,
245            "f1_macro": skm2.f1_score,
246            "f1_weighted": skm2.f1_score,
247            "f1_samples": skm2.f1_score,
248            "neg_log_loss": skm2.log_loss,
249            "precision": skm2.precision_score,
250            "recall": skm2.recall_score,
251            "roc_auc": skm2.roc_auc_score,
252        }
253
254        return scoring_options[scoring](y, preds, **kwargs)

Multitask Classification model based on regression models, with shared covariates

Parameters:

obj: object
    any object (must be a regression model) containing a method fit (obj.fit())
    and a method predict (obj.predict())

seed: int
    reproducibility seed

Attributes:

fit_objs_: dict
    objects adjusted to each individual time series

n_classes_: int
    number of classes for the classifier

Examples:

import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time

breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target

X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2,
                                                    random_state=123+2*10)

# Linear Regression is used
regr = LinearRegression()
fit_obj = ns.SimpleMultitaskClassifier(regr)

start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))

start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
SimpleMultitaskClassifier(obj)
76    def __init__(
77        self,
78        obj,
79    ):
80        self.type_fit = "classification"
81        self.obj = obj
82        self.fit_objs_ = {}
83        self.X_scaler_ = StandardScaler()
84        self.scaled_X_ = None
type_fit
obj
fit_objs_
X_scaler_
scaled_X_
def fit(self, X, y, sample_weight=None, **kwargs):
 86    def fit(self, X, y, sample_weight=None, **kwargs):
 87        """Fit SimpleMultitaskClassifier to training data (X, y).
 88
 89        Args:
 90
 91            X: {array-like}, shape = [n_samples, n_features]
 92                Training vectors, where n_samples is the number
 93                of samples and n_features is the number of features.
 94
 95            y: array-like, shape = [n_samples]
 96                Target values.
 97
 98            **kwargs: additional parameters to be passed to
 99                    self.cook_training_set or self.obj.fit
100
101        Returns:
102
103            self: object
104
105        """
106
107        assert mx.is_factor(y), "y must contain only integers"
108
109        self.scaled_X_ = self.X_scaler_.fit_transform(X)
110
111        self.n_classes_ = len(np.unique(y))
112
113        # multitask response
114        Y = mo.one_hot_encode2(y, self.n_classes_)
115
116        # if sample_weight is None:
117        for i in range(self.n_classes_):
118            self.fit_objs_[i] = deepcopy(
119                self.obj.fit(self.scaled_X_, Y[:, i], **kwargs)
120            )
121
122        return self

Fit SimpleMultitaskClassifier to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
124    def predict(self, X, **kwargs):
125        """Predict test data X.
126
127        Args:
128
129            X: {array-like}, shape = [n_samples, n_features]
130                Training vectors, where n_samples is the number
131                of samples and n_features is the number of features.
132
133            **kwargs: additional parameters
134
135        Returns:
136
137            model predictions: {array-like}
138
139        """
140
141        return np.argmax(self.predict_proba(X, **kwargs), axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
143    def predict_proba(self, X, **kwargs):
144        """Predict probabilities for test data X.
145
146        Args:
147
148            X: {array-like}, shape = [n_samples, n_features]
149                Training vectors, where n_samples is the number
150                of samples and n_features is the number of features.
151
152            **kwargs: additional parameters
153
154        Returns:
155
156            probability estimates for test data: {array-like}
157
158        """
159
160        shape_X = X.shape
161
162        probs = np.zeros((shape_X[0], self.n_classes_))
163
164        if len(shape_X) == 1:
165            n_features = shape_X[0]
166
167            new_X = mo.rbind(
168                X.reshape(1, n_features),
169                np.ones(n_features).reshape(1, n_features),
170            )
171
172            Z = self.X_scaler_.transform(new_X, **kwargs)
173
174            # loop on all the classes
175            for i in range(self.n_classes_):
176                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0]
177
178        else:
179            Z = self.X_scaler_.transform(X, **kwargs)
180
181            # loop on all the classes
182            for i in range(self.n_classes_):
183                probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)
184
185        expit_raw_probs = expit(probs)
186
187        return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None]

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters

Returns:

probability estimates for test data: {array-like}
def score(self, X, y, scoring=None, **kwargs):
189    def score(self, X, y, scoring=None, **kwargs):
190        """ Score the model on test set features X and response y. 
191
192        Args:
193        
194            X: {array-like}, shape = [n_samples, n_features]
195                Training vectors, where n_samples is the number 
196                of samples and n_features is the number of features
197
198            y: array-like, shape = [n_samples]
199                Target values
200
201            scoring: str
202                must be in ('explained_variance', 'neg_mean_absolute_error', \
203                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
204                            'neg_median_absolute_error', 'r2')
205            
206            **kwargs: additional parameters to be passed to scoring functions
207               
208        Returns: 
209        
210            model scores: {array-like}
211
212        """
213
214        preds = self.predict(X)
215
216        if scoring is None:
217            scoring = "accuracy"
218
219        # check inputs
220        assert scoring in (
221            "accuracy",
222            "average_precision",
223            "brier_score_loss",
224            "f1",
225            "f1_micro",
226            "f1_macro",
227            "f1_weighted",
228            "f1_samples",
229            "neg_log_loss",
230            "precision",
231            "recall",
232            "roc_auc",
233        ), "'scoring' should be in ('accuracy', 'average_precision', \
234                           'brier_score_loss', 'f1', 'f1_micro', \
235                           'f1_macro', 'f1_weighted',  'f1_samples', \
236                           'neg_log_loss', 'precision', 'recall', \
237                           'roc_auc')"
238
239        scoring_options = {
240            "accuracy": skm2.accuracy_score,
241            "average_precision": skm2.average_precision_score,
242            "brier_score_loss": skm2.brier_score_loss,
243            "f1": skm2.f1_score,
244            "f1_micro": skm2.f1_score,
245            "f1_macro": skm2.f1_score,
246            "f1_weighted": skm2.f1_score,
247            "f1_samples": skm2.f1_score,
248            "neg_log_loss": skm2.log_loss,
249            "precision": skm2.precision_score,
250            "recall": skm2.recall_score,
251            "roc_auc": skm2.roc_auc_score,
252        }
253
254        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                             'neg_mean_squared_error', 'neg_mean_squared_log_error',                             'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

class Optimizer:
  9class Optimizer:
 10    """Optimizer class
 11
 12    Attributes:
 13
 14        type_optim: str
 15            type of optimizer, (currently) either 'sgd' (stochastic minibatch gradient descent)
 16            or 'scd' (stochastic minibatch coordinate descent)
 17
 18        num_iters: int
 19            number of iterations of the optimizer
 20
 21        learning_rate: float
 22            step size
 23
 24        batch_prop: float
 25            proportion of the initial data used at each optimization step
 26
 27        learning_method: str
 28            "poly" - learning rate decreasing as a polynomial function
 29            of # of iterations (default)
 30            "exp" - learning rate decreasing as an exponential function
 31            of # of iterations
 32            "momentum" - gradient descent using momentum
 33
 34        randomization: str
 35            type of randomization applied at each step
 36            "strat" - stratified subsampling (default)
 37            "shuffle" - random subsampling
 38
 39        mass: float
 40            mass on velocity, for `method` == "momentum"
 41
 42        decay: float
 43            coefficient of decrease of the learning rate for
 44            `method` == "poly" and `method` == "exp"
 45
 46        tolerance: float
 47            early stopping parameter (convergence of loss function)
 48
 49        verbose: int
 50            controls verbosity of gradient descent
 51            0 - nothing is printed
 52            1 - a progress bar is printed
 53            2 - successive loss function values are printed
 54
 55    """
 56
 57    # construct the object -----
 58
 59    def __init__(
 60        self,
 61        type_optim="sgd",
 62        num_iters=100,
 63        learning_rate=0.01,
 64        batch_prop=1.0,
 65        learning_method="momentum",
 66        randomization="strat",
 67        mass=0.9,
 68        decay=0.1,
 69        tolerance=1e-3,
 70        verbose=1,
 71    ):
 72        self.type_optim = type_optim
 73        self.num_iters = num_iters
 74        self.learning_rate = learning_rate
 75        self.batch_prop = batch_prop
 76        self.learning_method = learning_method
 77        self.randomization = randomization
 78        self.mass = mass
 79        self.decay = decay
 80        self.tolerance = tolerance
 81        self.verbose = verbose
 82        self.opt = None
 83
 84    def fit(self, loss_func, response, x0, **kwargs):
 85        """Fit GLM model to training data (X, y).
 86
 87        Args:
 88
 89            loss_func: loss function
 90
 91            response: array-like, shape = [n_samples]
 92            target variable (used for subsampling)
 93
 94            x0: array-like, shape = [n_features]
 95                initial value provided to the optimizer
 96
 97            **kwargs: additional parameters to be passed to
 98                    loss function
 99
100        Returns:
101
102            self: object
103
104        """
105
106        if self.type_optim == "scd":
107            self.results = scd(
108                loss_func,
109                response=response,
110                x=x0,
111                num_iters=self.num_iters,
112                batch_prop=self.batch_prop,
113                learning_rate=self.learning_rate,
114                learning_method=self.learning_method,
115                mass=self.mass,
116                decay=self.decay,
117                randomization=self.randomization,
118                tolerance=self.tolerance,
119                verbose=self.verbose,
120                **kwargs
121            )
122
123        if self.type_optim == "sgd":
124            self.results = sgd(
125                loss_func,
126                response=response,
127                x=x0,
128                num_iters=self.num_iters,
129                batch_prop=self.batch_prop,
130                learning_rate=self.learning_rate,
131                learning_method=self.learning_method,
132                mass=self.mass,
133                decay=self.decay,
134                randomization=self.randomization,
135                tolerance=self.tolerance,
136                verbose=self.verbose,
137                **kwargs
138            )
139
140        return self
141
142    def one_hot_encode(self, y, n_classes):
143        return one_hot_encode(y, n_classes)

Optimizer class

Attributes:

type_optim: str
    type of optimizer, (currently) either 'sgd' (stochastic minibatch gradient descent)
    or 'scd' (stochastic minibatch coordinate descent)

num_iters: int
    number of iterations of the optimizer

learning_rate: float
    step size

batch_prop: float
    proportion of the initial data used at each optimization step

learning_method: str
    "poly" - learning rate decreasing as a polynomial function
    of # of iterations (default)
    "exp" - learning rate decreasing as an exponential function
    of # of iterations
    "momentum" - gradient descent using momentum

randomization: str
    type of randomization applied at each step
    "strat" - stratified subsampling (default)
    "shuffle" - random subsampling

mass: float
    mass on velocity, for `method` == "momentum"

decay: float
    coefficient of decrease of the learning rate for
    `method` == "poly" and `method` == "exp"

tolerance: float
    early stopping parameter (convergence of loss function)

verbose: int
    controls verbosity of gradient descent
    0 - nothing is printed
    1 - a progress bar is printed
    2 - successive loss function values are printed
Optimizer( type_optim='sgd', num_iters=100, learning_rate=0.01, batch_prop=1.0, learning_method='momentum', randomization='strat', mass=0.9, decay=0.1, tolerance=0.001, verbose=1)
59    def __init__(
60        self,
61        type_optim="sgd",
62        num_iters=100,
63        learning_rate=0.01,
64        batch_prop=1.0,
65        learning_method="momentum",
66        randomization="strat",
67        mass=0.9,
68        decay=0.1,
69        tolerance=1e-3,
70        verbose=1,
71    ):
72        self.type_optim = type_optim
73        self.num_iters = num_iters
74        self.learning_rate = learning_rate
75        self.batch_prop = batch_prop
76        self.learning_method = learning_method
77        self.randomization = randomization
78        self.mass = mass
79        self.decay = decay
80        self.tolerance = tolerance
81        self.verbose = verbose
82        self.opt = None
type_optim
num_iters
learning_rate
batch_prop
learning_method
randomization
mass
decay
tolerance
verbose
opt
def fit(self, loss_func, response, x0, **kwargs):
 84    def fit(self, loss_func, response, x0, **kwargs):
 85        """Fit GLM model to training data (X, y).
 86
 87        Args:
 88
 89            loss_func: loss function
 90
 91            response: array-like, shape = [n_samples]
 92            target variable (used for subsampling)
 93
 94            x0: array-like, shape = [n_features]
 95                initial value provided to the optimizer
 96
 97            **kwargs: additional parameters to be passed to
 98                    loss function
 99
100        Returns:
101
102            self: object
103
104        """
105
106        if self.type_optim == "scd":
107            self.results = scd(
108                loss_func,
109                response=response,
110                x=x0,
111                num_iters=self.num_iters,
112                batch_prop=self.batch_prop,
113                learning_rate=self.learning_rate,
114                learning_method=self.learning_method,
115                mass=self.mass,
116                decay=self.decay,
117                randomization=self.randomization,
118                tolerance=self.tolerance,
119                verbose=self.verbose,
120                **kwargs
121            )
122
123        if self.type_optim == "sgd":
124            self.results = sgd(
125                loss_func,
126                response=response,
127                x=x0,
128                num_iters=self.num_iters,
129                batch_prop=self.batch_prop,
130                learning_rate=self.learning_rate,
131                learning_method=self.learning_method,
132                mass=self.mass,
133                decay=self.decay,
134                randomization=self.randomization,
135                tolerance=self.tolerance,
136                verbose=self.verbose,
137                **kwargs
138            )
139
140        return self

Fit GLM model to training data (X, y).

Args:

loss_func: loss function

response: array-like, shape = [n_samples]
target variable (used for subsampling)

x0: array-like, shape = [n_features]
    initial value provided to the optimizer

**kwargs: additional parameters to be passed to
        loss function

Returns:

self: object
def one_hot_encode(self, y, n_classes):
142    def one_hot_encode(self, y, n_classes):
143        return one_hot_encode(y, n_classes)
class RandomBagRegressor(nnetsauce.randombag.bag.RandomBag, sklearn.base.RegressorMixin):
 18class RandomBagRegressor(RandomBag, RegressorMixin):
 19    """Randomized 'Bagging' Regression model
 20
 21    Parameters:
 22
 23        obj: object
 24            any object containing a method fit (obj.fit()) and a method predict
 25            (obj.predict())
 26
 27        n_estimators: int
 28            number of boosting iterations
 29
 30        n_hidden_features: int
 31            number of nodes in the hidden layer
 32
 33        activation_name: str
 34            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 35
 36        a: float
 37            hyperparameter for 'prelu' or 'elu' activation function
 38
 39        nodes_sim: str
 40            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 41            'uniform'
 42
 43        bias: boolean
 44            indicates if the hidden layer contains a bias term (True) or not
 45            (False)
 46
 47        dropout: float
 48            regularization parameter; (random) percentage of nodes dropped out
 49            of the training
 50
 51        direct_link: boolean
 52            indicates if the original predictors are included (True) in model''s
 53            fitting or not (False)
 54
 55        n_clusters: int
 56            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 57                no clustering)
 58
 59        cluster_encode: bool
 60            defines how the variable containing clusters is treated (default is one-hot)
 61            if `False`, then labels are used, without one-hot encoding
 62
 63        type_clust: str
 64            type of clustering method: currently k-means ('kmeans') or Gaussian
 65            Mixture Model ('gmm')
 66
 67        type_scaling: a tuple of 3 strings
 68            scaling methods for inputs, hidden layer, and clustering respectively
 69            (and when relevant).
 70            Currently available: standardization ('std') or MinMax scaling ('minmax')
 71
 72        col_sample: float
 73            percentage of covariates randomly chosen for training
 74
 75        row_sample: float
 76            percentage of rows chosen for training, by stratified bootstrapping
 77
 78        seed: int
 79            reproducibility seed for nodes_sim=='uniform'
 80
 81        backend: str
 82            "cpu" or "gpu" or "tpu"
 83
 84    Attributes:
 85
 86        voter_: dict
 87            dictionary containing all the fitted base-learners
 88
 89
 90    Examples:
 91
 92    ```python
 93    import numpy as np
 94    import nnetsauce as ns
 95    from sklearn.datasets import fetch_california_housing
 96    from sklearn.tree import DecisionTreeRegressor
 97    from sklearn.model_selection import train_test_split
 98
 99    X, y = fetch_california_housing(return_X_y=True, as_frame=False)
100
101    # split data into training test and test set
102    X_train, X_test, y_train, y_test = train_test_split(X, y,
103                                                        test_size=0.2, random_state=13)
104
105    # Requires further tuning
106    obj = DecisionTreeRegressor(max_depth=3, random_state=123)
107    obj2 = ns.RandomBagRegressor(obj=obj, direct_link=False,
108                                n_estimators=50,
109                                col_sample=0.9, row_sample=0.9,
110                                dropout=0, n_clusters=0, verbose=1)
111
112    obj2.fit(X_train, y_train)
113
114    print(np.sqrt(obj2.score(X_test, y_test))) # RMSE
115
116    ```
117
118    """
119
120    # construct the object -----
121
122    def __init__(
123        self,
124        obj,
125        n_estimators=10,
126        n_hidden_features=1,
127        activation_name="relu",
128        a=0.01,
129        nodes_sim="sobol",
130        bias=True,
131        dropout=0,
132        direct_link=False,
133        n_clusters=2,
134        cluster_encode=True,
135        type_clust="kmeans",
136        type_scaling=("std", "std", "std"),
137        col_sample=1,
138        row_sample=1,
139        n_jobs=None,
140        seed=123,
141        verbose=1,
142        backend="cpu",
143    ):
144        super().__init__(
145            obj=obj,
146            n_estimators=n_estimators,
147            n_hidden_features=n_hidden_features,
148            activation_name=activation_name,
149            a=a,
150            nodes_sim=nodes_sim,
151            bias=bias,
152            dropout=dropout,
153            direct_link=direct_link,
154            n_clusters=n_clusters,
155            cluster_encode=cluster_encode,
156            type_clust=type_clust,
157            type_scaling=type_scaling,
158            col_sample=col_sample,
159            row_sample=row_sample,
160            seed=seed,
161            backend=backend,
162        )
163
164        self.type_fit = "regression"
165        self.verbose = verbose
166        self.n_jobs = n_jobs
167        self.voter_ = {}
168
169    def fit(self, X, y, **kwargs):
170        """Fit Random 'Bagging' model to training data (X, y).
171
172        Args:
173
174            X: {array-like}, shape = [n_samples, n_features]
175                Training vectors, where n_samples is the number
176                of samples and n_features is the number of features.
177
178            y: array-like, shape = [n_samples]
179                Target values.
180
181            **kwargs: additional parameters to be passed to
182                    self.cook_training_set or self.obj.fit
183
184        Returns:
185
186            self: object
187
188        """
189
190        base_learner = CustomRegressor(
191            self.obj,
192            n_hidden_features=self.n_hidden_features,
193            activation_name=self.activation_name,
194            a=self.a,
195            nodes_sim=self.nodes_sim,
196            bias=self.bias,
197            dropout=self.dropout,
198            direct_link=self.direct_link,
199            n_clusters=self.n_clusters,
200            type_clust=self.type_clust,
201            type_scaling=self.type_scaling,
202            col_sample=self.col_sample,
203            row_sample=self.row_sample,
204            seed=self.seed,
205        )
206
207        # 1 - Sequential training -----
208
209        if self.n_jobs is None:
210            self.voter_ = rbagloop_regression(
211                base_learner, X, y, self.n_estimators, self.verbose, self.seed
212            )
213
214            self.n_estimators = len(self.voter_)
215
216            return self
217
218        # 2 - Parallel training -----
219        # buggy
220        # if self.n_jobs is not None:
221        def fit_estimators(m):
222            base_learner__ = pickle.loads(pickle.dumps(base_learner, -1))
223            base_learner__.set_params(seed=self.seed + m * 1000)
224            base_learner__.fit(X, y, **kwargs)
225            return base_learner__
226
227        if self.verbose == 1:
228            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
229                delayed(fit_estimators)(m)
230                for m in tqdm(range(self.n_estimators))
231            )
232        else:
233            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
234                delayed(fit_estimators)(m) for m in range(self.n_estimators)
235            )
236
237        self.voter_ = {i: elt for i, elt in enumerate(voters_list)}
238
239        self.n_estimators = len(self.voter_)
240
241        return self
242
243    def predict(self, X, weights=None, **kwargs):
244        """Predict for test data X.
245
246        Args:
247
248            X: {array-like}, shape = [n_samples, n_features]
249                Training vectors, where n_samples is the number
250                of samples and n_features is the number of features.
251
252            **kwargs: additional parameters to be passed to
253                    self.cook_test_set
254
255        Returns:
256
257            estimates for test data: {array-like}
258
259        """
260
261        def calculate_preds(voter, weights=None):
262            ensemble_preds = 0
263
264            n_iter = len(voter)
265
266            assert n_iter > 0, "no estimator found in `RandomBag` ensemble"
267
268            if weights is None:
269                for idx, elt in voter.items():
270                    ensemble_preds += elt.predict(X)
271
272                return ensemble_preds / n_iter
273
274            # if weights is not None:
275            for idx, elt in voter.items():
276                ensemble_preds += weights[idx] * elt.predict(X)
277
278            return ensemble_preds
279
280        # end calculate_preds ----
281
282        if weights is None:
283            return calculate_preds(self.voter_)
284
285        # if weights is not None:
286        self.weights = weights
287
288        return calculate_preds(self.voter_, weights)
289
290    def score(self, X, y, scoring=None, **kwargs):
291        """ Score the model on test set features X and response y. 
292
293        Args:
294        
295            X: {array-like}, shape = [n_samples, n_features]
296                Training vectors, where n_samples is the number 
297                of samples and n_features is the number of features
298
299            y: array-like, shape = [n_samples]
300                Target values
301
302            scoring: str
303                must be in ('explained_variance', 'neg_mean_absolute_error', \
304                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
305                            'neg_median_absolute_error', 'r2')
306            
307            **kwargs: additional parameters to be passed to scoring functions
308               
309        Returns: 
310        
311            model scores: {array-like}
312            
313        """
314
315        preds = self.predict(X)
316
317        if type(preds) == tuple:  # if there are std. devs in the predictions
318            preds = preds[0]
319
320        if scoring is None:
321            scoring = "neg_mean_squared_error"
322
323        # check inputs
324        assert scoring in (
325            "explained_variance",
326            "neg_mean_absolute_error",
327            "neg_mean_squared_error",
328            "neg_mean_squared_log_error",
329            "neg_median_absolute_error",
330            "r2",
331        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
332                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
333                           'neg_median_absolute_error', 'r2')"
334
335        scoring_options = {
336            "explained_variance": skm.explained_variance_score,
337            "neg_mean_absolute_error": skm.median_absolute_error,
338            "neg_mean_squared_error": skm.mean_squared_error,
339            "neg_mean_squared_log_error": skm.mean_squared_log_error,
340            "neg_median_absolute_error": skm.median_absolute_error,
341            "r2": skm.r2_score,
342        }
343
344        return scoring_options[scoring](y, preds, **kwargs)

Randomized 'Bagging' Regression model

Parameters:

obj: object
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict())

n_estimators: int
    number of boosting iterations

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model''s
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

voter_: dict
    dictionary containing all the fitted base-learners

Examples:

import numpy as np
import nnetsauce as ns
from sklearn.datasets import fetch_california_housing
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split

X, y = fetch_california_housing(return_X_y=True, as_frame=False)

# split data into training test and test set
X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                    test_size=0.2, random_state=13)

# Requires further tuning
obj = DecisionTreeRegressor(max_depth=3, random_state=123)
obj2 = ns.RandomBagRegressor(obj=obj, direct_link=False,
                            n_estimators=50,
                            col_sample=0.9, row_sample=0.9,
                            dropout=0, n_clusters=0, verbose=1)

obj2.fit(X_train, y_train)

print(np.sqrt(obj2.score(X_test, y_test))) # RMSE
RandomBagRegressor( obj, n_estimators=10, n_hidden_features=1, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=False, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, n_jobs=None, seed=123, verbose=1, backend='cpu')
122    def __init__(
123        self,
124        obj,
125        n_estimators=10,
126        n_hidden_features=1,
127        activation_name="relu",
128        a=0.01,
129        nodes_sim="sobol",
130        bias=True,
131        dropout=0,
132        direct_link=False,
133        n_clusters=2,
134        cluster_encode=True,
135        type_clust="kmeans",
136        type_scaling=("std", "std", "std"),
137        col_sample=1,
138        row_sample=1,
139        n_jobs=None,
140        seed=123,
141        verbose=1,
142        backend="cpu",
143    ):
144        super().__init__(
145            obj=obj,
146            n_estimators=n_estimators,
147            n_hidden_features=n_hidden_features,
148            activation_name=activation_name,
149            a=a,
150            nodes_sim=nodes_sim,
151            bias=bias,
152            dropout=dropout,
153            direct_link=direct_link,
154            n_clusters=n_clusters,
155            cluster_encode=cluster_encode,
156            type_clust=type_clust,
157            type_scaling=type_scaling,
158            col_sample=col_sample,
159            row_sample=row_sample,
160            seed=seed,
161            backend=backend,
162        )
163
164        self.type_fit = "regression"
165        self.verbose = verbose
166        self.n_jobs = n_jobs
167        self.voter_ = {}
type_fit
verbose
n_jobs
voter_
def fit(self, X, y, **kwargs):
169    def fit(self, X, y, **kwargs):
170        """Fit Random 'Bagging' model to training data (X, y).
171
172        Args:
173
174            X: {array-like}, shape = [n_samples, n_features]
175                Training vectors, where n_samples is the number
176                of samples and n_features is the number of features.
177
178            y: array-like, shape = [n_samples]
179                Target values.
180
181            **kwargs: additional parameters to be passed to
182                    self.cook_training_set or self.obj.fit
183
184        Returns:
185
186            self: object
187
188        """
189
190        base_learner = CustomRegressor(
191            self.obj,
192            n_hidden_features=self.n_hidden_features,
193            activation_name=self.activation_name,
194            a=self.a,
195            nodes_sim=self.nodes_sim,
196            bias=self.bias,
197            dropout=self.dropout,
198            direct_link=self.direct_link,
199            n_clusters=self.n_clusters,
200            type_clust=self.type_clust,
201            type_scaling=self.type_scaling,
202            col_sample=self.col_sample,
203            row_sample=self.row_sample,
204            seed=self.seed,
205        )
206
207        # 1 - Sequential training -----
208
209        if self.n_jobs is None:
210            self.voter_ = rbagloop_regression(
211                base_learner, X, y, self.n_estimators, self.verbose, self.seed
212            )
213
214            self.n_estimators = len(self.voter_)
215
216            return self
217
218        # 2 - Parallel training -----
219        # buggy
220        # if self.n_jobs is not None:
221        def fit_estimators(m):
222            base_learner__ = pickle.loads(pickle.dumps(base_learner, -1))
223            base_learner__.set_params(seed=self.seed + m * 1000)
224            base_learner__.fit(X, y, **kwargs)
225            return base_learner__
226
227        if self.verbose == 1:
228            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
229                delayed(fit_estimators)(m)
230                for m in tqdm(range(self.n_estimators))
231            )
232        else:
233            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
234                delayed(fit_estimators)(m) for m in range(self.n_estimators)
235            )
236
237        self.voter_ = {i: elt for i, elt in enumerate(voters_list)}
238
239        self.n_estimators = len(self.voter_)
240
241        return self

Fit Random 'Bagging' model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, weights=None, **kwargs):
243    def predict(self, X, weights=None, **kwargs):
244        """Predict for test data X.
245
246        Args:
247
248            X: {array-like}, shape = [n_samples, n_features]
249                Training vectors, where n_samples is the number
250                of samples and n_features is the number of features.
251
252            **kwargs: additional parameters to be passed to
253                    self.cook_test_set
254
255        Returns:
256
257            estimates for test data: {array-like}
258
259        """
260
261        def calculate_preds(voter, weights=None):
262            ensemble_preds = 0
263
264            n_iter = len(voter)
265
266            assert n_iter > 0, "no estimator found in `RandomBag` ensemble"
267
268            if weights is None:
269                for idx, elt in voter.items():
270                    ensemble_preds += elt.predict(X)
271
272                return ensemble_preds / n_iter
273
274            # if weights is not None:
275            for idx, elt in voter.items():
276                ensemble_preds += weights[idx] * elt.predict(X)
277
278            return ensemble_preds
279
280        # end calculate_preds ----
281
282        if weights is None:
283            return calculate_preds(self.voter_)
284
285        # if weights is not None:
286        self.weights = weights
287
288        return calculate_preds(self.voter_, weights)

Predict for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

estimates for test data: {array-like}
def score(self, X, y, scoring=None, **kwargs):
290    def score(self, X, y, scoring=None, **kwargs):
291        """ Score the model on test set features X and response y. 
292
293        Args:
294        
295            X: {array-like}, shape = [n_samples, n_features]
296                Training vectors, where n_samples is the number 
297                of samples and n_features is the number of features
298
299            y: array-like, shape = [n_samples]
300                Target values
301
302            scoring: str
303                must be in ('explained_variance', 'neg_mean_absolute_error', \
304                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
305                            'neg_median_absolute_error', 'r2')
306            
307            **kwargs: additional parameters to be passed to scoring functions
308               
309        Returns: 
310        
311            model scores: {array-like}
312            
313        """
314
315        preds = self.predict(X)
316
317        if type(preds) == tuple:  # if there are std. devs in the predictions
318            preds = preds[0]
319
320        if scoring is None:
321            scoring = "neg_mean_squared_error"
322
323        # check inputs
324        assert scoring in (
325            "explained_variance",
326            "neg_mean_absolute_error",
327            "neg_mean_squared_error",
328            "neg_mean_squared_log_error",
329            "neg_median_absolute_error",
330            "r2",
331        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
332                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
333                           'neg_median_absolute_error', 'r2')"
334
335        scoring_options = {
336            "explained_variance": skm.explained_variance_score,
337            "neg_mean_absolute_error": skm.median_absolute_error,
338            "neg_mean_squared_error": skm.mean_squared_error,
339            "neg_mean_squared_log_error": skm.mean_squared_log_error,
340            "neg_median_absolute_error": skm.median_absolute_error,
341            "r2": skm.r2_score,
342        }
343
344        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                             'neg_mean_squared_error', 'neg_mean_squared_log_error',                             'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_predict_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.randombag.bag.RandomBag
obj
n_estimators
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
class RandomBagClassifier(nnetsauce.randombag.bag.RandomBag, sklearn.base.ClassifierMixin):
 18class RandomBagClassifier(RandomBag, ClassifierMixin):
 19    """Randomized 'Bagging' Classification model
 20
 21    Parameters:
 22
 23        obj: object
 24            any object containing a method fit (obj.fit()) and a method predict
 25            (obj.predict())
 26
 27        n_estimators: int
 28            number of boosting iterations
 29
 30        n_hidden_features: int
 31            number of nodes in the hidden layer
 32
 33        activation_name: str
 34            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 35
 36        a: float
 37            hyperparameter for 'prelu' or 'elu' activation function
 38
 39        nodes_sim: str
 40            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 41            'uniform'
 42
 43        bias: boolean
 44            indicates if the hidden layer contains a bias term (True) or not
 45            (False)
 46
 47        dropout: float
 48            regularization parameter; (random) percentage of nodes dropped out
 49            of the training
 50
 51        direct_link: boolean
 52            indicates if the original predictors are included (True) in model's
 53            fitting or not (False)
 54
 55        n_clusters: int
 56            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 57                no clustering)
 58
 59        cluster_encode: bool
 60            defines how the variable containing clusters is treated (default is one-hot)
 61            if `False`, then labels are used, without one-hot encoding
 62
 63        type_clust: str
 64            type of clustering method: currently k-means ('kmeans') or Gaussian
 65            Mixture Model ('gmm')
 66
 67        type_scaling: a tuple of 3 strings
 68            scaling methods for inputs, hidden layer, and clustering respectively
 69            (and when relevant).
 70            Currently available: standardization ('std') or MinMax scaling ('minmax')
 71
 72        col_sample: float
 73            percentage of covariates randomly chosen for training
 74
 75        row_sample: float
 76            percentage of rows chosen for training, by stratified bootstrapping
 77
 78        seed: int
 79            reproducibility seed for nodes_sim=='uniform'
 80
 81        backend: str
 82            "cpu" or "gpu" or "tpu"
 83
 84    Attributes:
 85
 86        voter_: dict
 87            dictionary containing all the fitted base-learners
 88
 89
 90    Examples:
 91
 92    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/randombag_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/randombag_classification.py)
 93
 94    ```python
 95    import nnetsauce as ns
 96    from sklearn.datasets import load_breast_cancer
 97    from sklearn.tree import DecisionTreeClassifier
 98    from sklearn.model_selection import train_test_split
 99    from sklearn import metrics
100    from time import time
101
102
103    breast_cancer = load_breast_cancer()
104    Z = breast_cancer.data
105    t = breast_cancer.target
106    np.random.seed(123)
107    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
108
109    # decision tree
110    clf = DecisionTreeClassifier(max_depth=2, random_state=123)
111    fit_obj = ns.RandomBagClassifier(clf, n_hidden_features=2,
112                                    direct_link=True,
113                                    n_estimators=100,
114                                    col_sample=0.9, row_sample=0.9,
115                                    dropout=0.3, n_clusters=0, verbose=1)
116
117    start = time()
118    fit_obj.fit(X_train, y_train)
119    print(f"Elapsed {time() - start}")
120
121    print(fit_obj.score(X_test, y_test))
122    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
123
124    start = time()
125    preds = fit_obj.predict(X_test)
126    print(f"Elapsed {time() - start}")
127    print(metrics.classification_report(preds, y_test))
128    ```
129
130    """
131
132    # construct the object -----
133
134    def __init__(
135        self,
136        obj,
137        n_estimators=10,
138        n_hidden_features=1,
139        activation_name="relu",
140        a=0.01,
141        nodes_sim="sobol",
142        bias=True,
143        dropout=0,
144        direct_link=False,
145        n_clusters=2,
146        cluster_encode=True,
147        type_clust="kmeans",
148        type_scaling=("std", "std", "std"),
149        col_sample=1,
150        row_sample=1,
151        n_jobs=None,
152        seed=123,
153        verbose=1,
154        backend="cpu",
155    ):
156        super().__init__(
157            obj=obj,
158            n_estimators=n_estimators,
159            n_hidden_features=n_hidden_features,
160            activation_name=activation_name,
161            a=a,
162            nodes_sim=nodes_sim,
163            bias=bias,
164            dropout=dropout,
165            direct_link=direct_link,
166            n_clusters=n_clusters,
167            cluster_encode=cluster_encode,
168            type_clust=type_clust,
169            type_scaling=type_scaling,
170            col_sample=col_sample,
171            row_sample=row_sample,
172            seed=seed,
173            backend=backend,
174        )
175
176        self.type_fit = "classification"
177        self.verbose = verbose
178        self.n_jobs = n_jobs
179        self.voter_ = {}
180
181    def fit(self, X, y, **kwargs):
182        """Fit Random 'Bagging' model to training data (X, y).
183
184        Args:
185
186            X: {array-like}, shape = [n_samples, n_features]
187                Training vectors, where n_samples is the number
188                of samples and n_features is the number of features.
189
190            y: array-like, shape = [n_samples]
191                Target values.
192
193            **kwargs: additional parameters to be passed to
194                    self.cook_training_set or self.obj.fit
195
196        Returns:
197
198            self: object
199
200        """
201
202        assert mx.is_factor(y), "y must contain only integers"
203
204        # training
205        self.n_classes = len(np.unique(y))
206
207        base_learner = CustomClassifier(
208            self.obj,
209            n_hidden_features=self.n_hidden_features,
210            activation_name=self.activation_name,
211            a=self.a,
212            nodes_sim=self.nodes_sim,
213            bias=self.bias,
214            dropout=self.dropout,
215            direct_link=self.direct_link,
216            n_clusters=self.n_clusters,
217            type_clust=self.type_clust,
218            type_scaling=self.type_scaling,
219            col_sample=self.col_sample,
220            row_sample=self.row_sample,
221            seed=self.seed,
222        )
223
224        # 1 - Sequential training -----
225
226        if self.n_jobs is None:
227            self.voter_ = rbagloop_classification(
228                base_learner, X, y, self.n_estimators, self.verbose, self.seed
229            )
230
231            self.n_estimators = len(self.voter_)
232
233            return self
234
235        # 2 - Parallel training -----
236        # buggy
237        # if self.n_jobs is not None:
238        def fit_estimators(m):
239            base_learner__ = pickle.loads(pickle.dumps(base_learner, -1))
240            base_learner__.set_params(seed=self.seed + m * 1000)
241            base_learner__.fit(X, y, **kwargs)
242            return base_learner__
243
244        if self.verbose == 1:
245            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
246                delayed(fit_estimators)(m)
247                for m in tqdm(range(self.n_estimators))
248            )
249        else:
250            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
251                delayed(fit_estimators)(m) for m in range(self.n_estimators)
252            )
253
254        self.voter_ = {idx: elt for idx, elt in enumerate(voters_list)}
255
256        self.n_estimators = len(self.voter_)
257
258        return self
259
260    def predict(self, X, weights=None, **kwargs):
261        """Predict test data X.
262
263        Args:
264
265            X: {array-like}, shape = [n_samples, n_features]
266                Training vectors, where n_samples is the number
267                of samples and n_features is the number of features.
268
269            **kwargs: additional parameters to be passed to
270                    self.cook_test_set
271
272        Returns:
273
274            model predictions: {array-like}
275
276        """
277        return self.predict_proba(X, weights, **kwargs).argmax(axis=1)
278
279    def predict_proba(self, X, weights=None, **kwargs):
280        """Predict probabilities for test data X.
281
282        Args:
283
284            X: {array-like}, shape = [n_samples, n_features]
285                Training vectors, where n_samples is the number
286                of samples and n_features is the number of features.
287
288            **kwargs: additional parameters to be passed to
289                    self.cook_test_set
290
291        Returns:
292
293            probability estimates for test data: {array-like}
294
295        """
296
297        def calculate_probas(voter, weights=None, verbose=None):
298            ensemble_proba = 0
299
300            n_iter = len(voter)
301
302            assert n_iter > 0, "no estimator found in `RandomBag` ensemble"
303
304            if weights is None:
305                for idx, elt in voter.items():
306                    try:
307                        ensemble_proba += elt.predict_proba(X)
308
309                        # if verbose == 1:
310                        #    pbar.update(idx)
311
312                    except:
313                        continue
314
315                # if verbose == 1:
316                #    pbar.update(n_iter)
317
318                return ensemble_proba / n_iter
319
320            # if weights is not None:
321            for idx, elt in voter.items():
322                ensemble_proba += weights[idx] * elt.predict_proba(X)
323
324                # if verbose == 1:
325                #    pbar.update(idx)
326
327            # if verbose == 1:
328            #    pbar.update(n_iter)
329
330            return ensemble_proba
331
332        # end calculate_probas ----
333
334        if self.n_jobs is None:
335            # if self.verbose == 1:
336            #    pbar = Progbar(self.n_estimators)
337
338            if weights is None:
339                return calculate_probas(self.voter_, verbose=self.verbose)
340
341            # if weights is not None:
342            self.weights = weights
343
344            return calculate_probas(self.voter_, weights, verbose=self.verbose)
345
346        # if self.n_jobs is not None:
347        def predict_estimator(m):
348            try:
349                return self.voter_[m].predict_proba(X)
350            except:
351                pass
352
353        if self.verbose == 1:
354            preds = Parallel(n_jobs=self.n_jobs, prefer="threads")(
355                delayed(predict_estimator)(m)
356                for m in tqdm(range(self.n_estimators))
357            )
358
359        else:
360            preds = Parallel(n_jobs=self.n_jobs, prefer="threads")(
361                delayed(predict_estimator)(m) for m in range(self.n_estimators)
362            )
363
364        ensemble_proba = 0
365
366        if weights is None:
367            for i in range(self.n_estimators):
368                ensemble_proba += preds[i]
369
370            return ensemble_proba / self.n_estimators
371
372        for i in range(self.n_estimators):
373            ensemble_proba += weights[i] * preds[i]
374
375        return ensemble_proba
376
377    def score(self, X, y, weights=None, scoring=None, **kwargs):
378        """ Score the model on test set features X and response y. 
379
380        Args:
381        
382            X: {array-like}, shape = [n_samples, n_features]
383                Training vectors, where n_samples is the number 
384                of samples and n_features is the number of features
385
386            y: array-like, shape = [n_samples]
387                Target values
388
389            scoring: str
390                must be in ('explained_variance', 'neg_mean_absolute_error', \
391                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
392                            'neg_median_absolute_error', 'r2')
393            
394            **kwargs: additional parameters to be passed to scoring functions
395               
396        Returns: 
397        
398            model scores: {array-like}
399
400        """
401
402        preds = self.predict(X, weights, **kwargs)
403
404        if scoring is None:
405            scoring = "accuracy"
406
407        # check inputs
408        assert scoring in (
409            "accuracy",
410            "average_precision",
411            "brier_score_loss",
412            "f1",
413            "f1_micro",
414            "f1_macro",
415            "f1_weighted",
416            "f1_samples",
417            "neg_log_loss",
418            "precision",
419            "recall",
420            "roc_auc",
421        ), "'scoring' should be in ('accuracy', 'average_precision', \
422                           'brier_score_loss', 'f1', 'f1_micro', \
423                           'f1_macro', 'f1_weighted',  'f1_samples', \
424                           'neg_log_loss', 'precision', 'recall', \
425                           'roc_auc')"
426
427        scoring_options = {
428            "accuracy": skm2.accuracy_score,
429            "average_precision": skm2.average_precision_score,
430            "brier_score_loss": skm2.brier_score_loss,
431            "f1": skm2.f1_score,
432            "f1_micro": skm2.f1_score,
433            "f1_macro": skm2.f1_score,
434            "f1_weighted": skm2.f1_score,
435            "f1_samples": skm2.f1_score,
436            "neg_log_loss": skm2.log_loss,
437            "precision": skm2.precision_score,
438            "recall": skm2.recall_score,
439            "roc_auc": skm2.roc_auc_score,
440        }
441
442        return scoring_options[scoring](y, preds, **kwargs)

Randomized 'Bagging' Classification model

Parameters:

obj: object
    any object containing a method fit (obj.fit()) and a method predict
    (obj.predict())

n_estimators: int
    number of boosting iterations

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

col_sample: float
    percentage of covariates randomly chosen for training

row_sample: float
    percentage of rows chosen for training, by stratified bootstrapping

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

voter_: dict
    dictionary containing all the fitted base-learners

Examples:

See also https://github.com/Techtonique/nnetsauce/blob/master/examples/randombag_classification.py

import nnetsauce as ns
from sklearn.datasets import load_breast_cancer
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time


breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)

# decision tree
clf = DecisionTreeClassifier(max_depth=2, random_state=123)
fit_obj = ns.RandomBagClassifier(clf, n_hidden_features=2,
                                direct_link=True,
                                n_estimators=100,
                                col_sample=0.9, row_sample=0.9,
                                dropout=0.3, n_clusters=0, verbose=1)

start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))

start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
RandomBagClassifier( obj, n_estimators=10, n_hidden_features=1, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=False, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), col_sample=1, row_sample=1, n_jobs=None, seed=123, verbose=1, backend='cpu')
134    def __init__(
135        self,
136        obj,
137        n_estimators=10,
138        n_hidden_features=1,
139        activation_name="relu",
140        a=0.01,
141        nodes_sim="sobol",
142        bias=True,
143        dropout=0,
144        direct_link=False,
145        n_clusters=2,
146        cluster_encode=True,
147        type_clust="kmeans",
148        type_scaling=("std", "std", "std"),
149        col_sample=1,
150        row_sample=1,
151        n_jobs=None,
152        seed=123,
153        verbose=1,
154        backend="cpu",
155    ):
156        super().__init__(
157            obj=obj,
158            n_estimators=n_estimators,
159            n_hidden_features=n_hidden_features,
160            activation_name=activation_name,
161            a=a,
162            nodes_sim=nodes_sim,
163            bias=bias,
164            dropout=dropout,
165            direct_link=direct_link,
166            n_clusters=n_clusters,
167            cluster_encode=cluster_encode,
168            type_clust=type_clust,
169            type_scaling=type_scaling,
170            col_sample=col_sample,
171            row_sample=row_sample,
172            seed=seed,
173            backend=backend,
174        )
175
176        self.type_fit = "classification"
177        self.verbose = verbose
178        self.n_jobs = n_jobs
179        self.voter_ = {}
type_fit
verbose
n_jobs
voter_
def fit(self, X, y, **kwargs):
181    def fit(self, X, y, **kwargs):
182        """Fit Random 'Bagging' model to training data (X, y).
183
184        Args:
185
186            X: {array-like}, shape = [n_samples, n_features]
187                Training vectors, where n_samples is the number
188                of samples and n_features is the number of features.
189
190            y: array-like, shape = [n_samples]
191                Target values.
192
193            **kwargs: additional parameters to be passed to
194                    self.cook_training_set or self.obj.fit
195
196        Returns:
197
198            self: object
199
200        """
201
202        assert mx.is_factor(y), "y must contain only integers"
203
204        # training
205        self.n_classes = len(np.unique(y))
206
207        base_learner = CustomClassifier(
208            self.obj,
209            n_hidden_features=self.n_hidden_features,
210            activation_name=self.activation_name,
211            a=self.a,
212            nodes_sim=self.nodes_sim,
213            bias=self.bias,
214            dropout=self.dropout,
215            direct_link=self.direct_link,
216            n_clusters=self.n_clusters,
217            type_clust=self.type_clust,
218            type_scaling=self.type_scaling,
219            col_sample=self.col_sample,
220            row_sample=self.row_sample,
221            seed=self.seed,
222        )
223
224        # 1 - Sequential training -----
225
226        if self.n_jobs is None:
227            self.voter_ = rbagloop_classification(
228                base_learner, X, y, self.n_estimators, self.verbose, self.seed
229            )
230
231            self.n_estimators = len(self.voter_)
232
233            return self
234
235        # 2 - Parallel training -----
236        # buggy
237        # if self.n_jobs is not None:
238        def fit_estimators(m):
239            base_learner__ = pickle.loads(pickle.dumps(base_learner, -1))
240            base_learner__.set_params(seed=self.seed + m * 1000)
241            base_learner__.fit(X, y, **kwargs)
242            return base_learner__
243
244        if self.verbose == 1:
245            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
246                delayed(fit_estimators)(m)
247                for m in tqdm(range(self.n_estimators))
248            )
249        else:
250            voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")(
251                delayed(fit_estimators)(m) for m in range(self.n_estimators)
252            )
253
254        self.voter_ = {idx: elt for idx, elt in enumerate(voters_list)}
255
256        self.n_estimators = len(self.voter_)
257
258        return self

Fit Random 'Bagging' model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, weights=None, **kwargs):
260    def predict(self, X, weights=None, **kwargs):
261        """Predict test data X.
262
263        Args:
264
265            X: {array-like}, shape = [n_samples, n_features]
266                Training vectors, where n_samples is the number
267                of samples and n_features is the number of features.
268
269            **kwargs: additional parameters to be passed to
270                    self.cook_test_set
271
272        Returns:
273
274            model predictions: {array-like}
275
276        """
277        return self.predict_proba(X, weights, **kwargs).argmax(axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, weights=None, **kwargs):
279    def predict_proba(self, X, weights=None, **kwargs):
280        """Predict probabilities for test data X.
281
282        Args:
283
284            X: {array-like}, shape = [n_samples, n_features]
285                Training vectors, where n_samples is the number
286                of samples and n_features is the number of features.
287
288            **kwargs: additional parameters to be passed to
289                    self.cook_test_set
290
291        Returns:
292
293            probability estimates for test data: {array-like}
294
295        """
296
297        def calculate_probas(voter, weights=None, verbose=None):
298            ensemble_proba = 0
299
300            n_iter = len(voter)
301
302            assert n_iter > 0, "no estimator found in `RandomBag` ensemble"
303
304            if weights is None:
305                for idx, elt in voter.items():
306                    try:
307                        ensemble_proba += elt.predict_proba(X)
308
309                        # if verbose == 1:
310                        #    pbar.update(idx)
311
312                    except:
313                        continue
314
315                # if verbose == 1:
316                #    pbar.update(n_iter)
317
318                return ensemble_proba / n_iter
319
320            # if weights is not None:
321            for idx, elt in voter.items():
322                ensemble_proba += weights[idx] * elt.predict_proba(X)
323
324                # if verbose == 1:
325                #    pbar.update(idx)
326
327            # if verbose == 1:
328            #    pbar.update(n_iter)
329
330            return ensemble_proba
331
332        # end calculate_probas ----
333
334        if self.n_jobs is None:
335            # if self.verbose == 1:
336            #    pbar = Progbar(self.n_estimators)
337
338            if weights is None:
339                return calculate_probas(self.voter_, verbose=self.verbose)
340
341            # if weights is not None:
342            self.weights = weights
343
344            return calculate_probas(self.voter_, weights, verbose=self.verbose)
345
346        # if self.n_jobs is not None:
347        def predict_estimator(m):
348            try:
349                return self.voter_[m].predict_proba(X)
350            except:
351                pass
352
353        if self.verbose == 1:
354            preds = Parallel(n_jobs=self.n_jobs, prefer="threads")(
355                delayed(predict_estimator)(m)
356                for m in tqdm(range(self.n_estimators))
357            )
358
359        else:
360            preds = Parallel(n_jobs=self.n_jobs, prefer="threads")(
361                delayed(predict_estimator)(m) for m in range(self.n_estimators)
362            )
363
364        ensemble_proba = 0
365
366        if weights is None:
367            for i in range(self.n_estimators):
368                ensemble_proba += preds[i]
369
370            return ensemble_proba / self.n_estimators
371
372        for i in range(self.n_estimators):
373            ensemble_proba += weights[i] * preds[i]
374
375        return ensemble_proba

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
def score(self, X, y, weights=None, scoring=None, **kwargs):
377    def score(self, X, y, weights=None, scoring=None, **kwargs):
378        """ Score the model on test set features X and response y. 
379
380        Args:
381        
382            X: {array-like}, shape = [n_samples, n_features]
383                Training vectors, where n_samples is the number 
384                of samples and n_features is the number of features
385
386            y: array-like, shape = [n_samples]
387                Target values
388
389            scoring: str
390                must be in ('explained_variance', 'neg_mean_absolute_error', \
391                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
392                            'neg_median_absolute_error', 'r2')
393            
394            **kwargs: additional parameters to be passed to scoring functions
395               
396        Returns: 
397        
398            model scores: {array-like}
399
400        """
401
402        preds = self.predict(X, weights, **kwargs)
403
404        if scoring is None:
405            scoring = "accuracy"
406
407        # check inputs
408        assert scoring in (
409            "accuracy",
410            "average_precision",
411            "brier_score_loss",
412            "f1",
413            "f1_micro",
414            "f1_macro",
415            "f1_weighted",
416            "f1_samples",
417            "neg_log_loss",
418            "precision",
419            "recall",
420            "roc_auc",
421        ), "'scoring' should be in ('accuracy', 'average_precision', \
422                           'brier_score_loss', 'f1', 'f1_micro', \
423                           'f1_macro', 'f1_weighted',  'f1_samples', \
424                           'neg_log_loss', 'precision', 'recall', \
425                           'roc_auc')"
426
427        scoring_options = {
428            "accuracy": skm2.accuracy_score,
429            "average_precision": skm2.average_precision_score,
430            "brier_score_loss": skm2.brier_score_loss,
431            "f1": skm2.f1_score,
432            "f1_micro": skm2.f1_score,
433            "f1_macro": skm2.f1_score,
434            "f1_weighted": skm2.f1_score,
435            "f1_samples": skm2.f1_score,
436            "neg_log_loss": skm2.log_loss,
437            "precision": skm2.precision_score,
438            "recall": skm2.recall_score,
439            "roc_auc": skm2.roc_auc_score,
440        }
441
442        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                             'neg_mean_squared_error', 'neg_mean_squared_log_error',                             'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_predict_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_predict_proba_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.randombag.bag.RandomBag
obj
n_estimators
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
class Ridge2Regressor(nnetsauce.ridge2.ridge2.Ridge2, sklearn.base.RegressorMixin):
 21class Ridge2Regressor(Ridge2, RegressorMixin):
 22    """Ridge regression with 2 regularization parameters derived from class Ridge
 23
 24    Parameters:
 25
 26        n_hidden_features: int
 27            number of nodes in the hidden layer
 28
 29        activation_name: str
 30            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 31
 32        a: float
 33            hyperparameter for 'prelu' or 'elu' activation function
 34
 35        nodes_sim: str
 36            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 37            'uniform'
 38
 39        bias: boolean
 40            indicates if the hidden layer contains a bias term (True) or not
 41            (False)
 42
 43        dropout: float
 44            regularization parameter; (random) percentage of nodes dropped out
 45            of the training
 46
 47        n_clusters: int
 48            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 49                no clustering)
 50
 51        cluster_encode: bool
 52            defines how the variable containing clusters is treated (default is one-hot)
 53            if `False`, then labels are used, without one-hot encoding
 54
 55        type_clust: str
 56            type of clustering method: currently k-means ('kmeans') or Gaussian
 57            Mixture Model ('gmm')
 58
 59        type_scaling: a tuple of 3 strings
 60            scaling methods for inputs, hidden layer, and clustering respectively
 61            (and when relevant).
 62            Currently available: standardization ('std') or MinMax scaling ('minmax')
 63
 64        lambda1: float
 65            regularization parameter on direct link
 66
 67        lambda2: float
 68            regularization parameter on hidden layer
 69
 70        seed: int
 71            reproducibility seed for nodes_sim=='uniform'
 72
 73        backend: str
 74            'cpu' or 'gpu' or 'tpu'
 75
 76    Attributes:
 77
 78        beta_: {array-like}
 79            regression coefficients
 80
 81        y_mean_: float
 82            average response
 83
 84    """
 85
 86    # construct the object -----
 87
 88    def __init__(
 89        self,
 90        n_hidden_features=5,
 91        activation_name="relu",
 92        a=0.01,
 93        nodes_sim="sobol",
 94        bias=True,
 95        dropout=0,
 96        n_clusters=2,
 97        cluster_encode=True,
 98        type_clust="kmeans",
 99        type_scaling=("std", "std", "std"),
100        lambda1=0.1,
101        lambda2=0.1,
102        seed=123,
103        backend="cpu",
104    ):
105        super().__init__(
106            n_hidden_features=n_hidden_features,
107            activation_name=activation_name,
108            a=a,
109            nodes_sim=nodes_sim,
110            bias=bias,
111            dropout=dropout,
112            n_clusters=n_clusters,
113            cluster_encode=cluster_encode,
114            type_clust=type_clust,
115            type_scaling=type_scaling,
116            lambda1=lambda1,
117            lambda2=lambda2,
118            seed=seed,
119            backend=backend,
120        )
121
122        self.type_fit = "regression"
123
124    def fit(self, X, y, **kwargs):
125        """Fit Ridge model to training data (X, y).
126
127        Args:
128
129            X: {array-like}, shape = [n_samples, n_features]
130                Training vectors, where n_samples is the number
131                of samples and n_features is the number of features.
132
133            y: array-like, shape = [n_samples]
134                Target values.
135
136            **kwargs: additional parameters to be passed to
137                    self.cook_training_set or self.obj.fit
138
139        Returns:
140
141            self: object
142
143        """
144
145        sys_platform = platform.system()
146
147        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
148
149        n_X, p_X = X.shape
150        n_Z, p_Z = scaled_Z.shape
151
152        if self.n_clusters > 0:
153            if self.encode_clusters == True:
154                n_features = p_X + self.n_clusters
155            else:
156                n_features = p_X + 1
157        else:
158            n_features = p_X
159
160        X_ = scaled_Z[:, 0:n_features]
161        Phi_X_ = scaled_Z[:, n_features:p_Z]
162
163        B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag(
164            np.repeat(1, n_features)
165        )
166        C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend)
167        D = mo.crossprod(
168            x=Phi_X_, backend=self.backend
169        ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1]))
170
171        if sys_platform in ("Linux", "Darwin"):
172            B_inv = pinv(B) if self.backend == "cpu" else jpinv(B)
173        else:
174            B_inv = pinv(B)
175
176        W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend)
177        S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend)
178
179        if sys_platform in ("Linux", "Darwin"):
180            S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat)
181        else:
182            S_inv = pinv(S_mat)
183
184        Y = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend)
185        inv = mo.rbind(
186            mo.cbind(
187                x=B_inv + mo.crossprod(x=W, y=Y, backend=self.backend),
188                y=-np.transpose(Y),
189                backend=self.backend,
190            ),
191            mo.cbind(x=-Y, y=S_inv, backend=self.backend),
192            backend=self.backend,
193        )
194
195        self.beta_ = mo.safe_sparse_dot(
196            a=inv,
197            b=mo.crossprod(x=scaled_Z, y=centered_y, backend=self.backend),
198            backend=self.backend,
199        )
200
201        return self
202
203    def predict(self, X, **kwargs):
204        """Predict test data X.
205
206        Args:
207
208            X: {array-like}, shape = [n_samples, n_features]
209                Training vectors, where n_samples is the number
210                of samples and n_features is the number of features.
211
212            **kwargs: additional parameters to be passed to
213                    self.cook_test_set
214
215        Returns:
216
217            model predictions: {array-like}
218
219        """
220
221        if len(X.shape) == 1:
222            n_features = X.shape[0]
223            new_X = mo.rbind(
224                x=X.reshape(1, n_features),
225                y=np.ones(n_features).reshape(1, n_features),
226                backend=self.backend,
227            )
228
229            return (
230                self.y_mean_
231                + mo.safe_sparse_dot(
232                    a=self.cook_test_set(new_X, **kwargs),
233                    b=self.beta_,
234                    backend=self.backend,
235                )
236            )[0]
237
238        return self.y_mean_ + mo.safe_sparse_dot(
239            a=self.cook_test_set(X, **kwargs),
240            b=self.beta_,
241            backend=self.backend,
242        )
243
244    def score(self, X, y, scoring=None, **kwargs):
245        """ Score the model on test set features X and response y. 
246
247        Args:
248        
249            X: {array-like}, shape = [n_samples, n_features]
250                Training vectors, where n_samples is the number 
251                of samples and n_features is the number of features
252
253            y: array-like, shape = [n_samples]
254                Target values
255
256            scoring: str
257                must be in ('explained_variance', 'neg_mean_absolute_error', \
258                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
259                            'neg_median_absolute_error', 'r2')
260            
261            **kwargs: additional parameters to be passed to scoring functions
262               
263        Returns: 
264        
265            model scores: {array-like}
266            
267        """
268
269        preds = self.predict(X)
270
271        if type(preds) == tuple:  # if there are std. devs in the predictions
272            preds = preds[0]
273
274        if scoring is None:
275            scoring = "neg_mean_squared_error"
276
277        # check inputs
278        assert scoring in (
279            "explained_variance",
280            "neg_mean_absolute_error",
281            "neg_mean_squared_error",
282            "neg_mean_squared_log_error",
283            "neg_median_absolute_error",
284            "r2",
285        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
286                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
287                           'neg_median_absolute_error', 'r2')"
288
289        scoring_options = {
290            "explained_variance": skm.explained_variance_score,
291            "neg_mean_absolute_error": skm.median_absolute_error,
292            "neg_mean_squared_error": skm.mean_squared_error,
293            "neg_mean_squared_log_error": skm.mean_squared_log_error,
294            "neg_median_absolute_error": skm.median_absolute_error,
295            "r2": skm.r2_score,
296        }
297
298        return scoring_options[scoring](y, preds, **kwargs)

Ridge regression with 2 regularization parameters derived from class Ridge

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

lambda1: float
    regularization parameter on direct link

lambda2: float
    regularization parameter on hidden layer

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    'cpu' or 'gpu' or 'tpu'

Attributes:

beta_: {array-like}
    regression coefficients

y_mean_: float
    average response
Ridge2Regressor( n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), lambda1=0.1, lambda2=0.1, seed=123, backend='cpu')
 88    def __init__(
 89        self,
 90        n_hidden_features=5,
 91        activation_name="relu",
 92        a=0.01,
 93        nodes_sim="sobol",
 94        bias=True,
 95        dropout=0,
 96        n_clusters=2,
 97        cluster_encode=True,
 98        type_clust="kmeans",
 99        type_scaling=("std", "std", "std"),
100        lambda1=0.1,
101        lambda2=0.1,
102        seed=123,
103        backend="cpu",
104    ):
105        super().__init__(
106            n_hidden_features=n_hidden_features,
107            activation_name=activation_name,
108            a=a,
109            nodes_sim=nodes_sim,
110            bias=bias,
111            dropout=dropout,
112            n_clusters=n_clusters,
113            cluster_encode=cluster_encode,
114            type_clust=type_clust,
115            type_scaling=type_scaling,
116            lambda1=lambda1,
117            lambda2=lambda2,
118            seed=seed,
119            backend=backend,
120        )
121
122        self.type_fit = "regression"
type_fit
def fit(self, X, y, **kwargs):
124    def fit(self, X, y, **kwargs):
125        """Fit Ridge model to training data (X, y).
126
127        Args:
128
129            X: {array-like}, shape = [n_samples, n_features]
130                Training vectors, where n_samples is the number
131                of samples and n_features is the number of features.
132
133            y: array-like, shape = [n_samples]
134                Target values.
135
136            **kwargs: additional parameters to be passed to
137                    self.cook_training_set or self.obj.fit
138
139        Returns:
140
141            self: object
142
143        """
144
145        sys_platform = platform.system()
146
147        centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
148
149        n_X, p_X = X.shape
150        n_Z, p_Z = scaled_Z.shape
151
152        if self.n_clusters > 0:
153            if self.encode_clusters == True:
154                n_features = p_X + self.n_clusters
155            else:
156                n_features = p_X + 1
157        else:
158            n_features = p_X
159
160        X_ = scaled_Z[:, 0:n_features]
161        Phi_X_ = scaled_Z[:, n_features:p_Z]
162
163        B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag(
164            np.repeat(1, n_features)
165        )
166        C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend)
167        D = mo.crossprod(
168            x=Phi_X_, backend=self.backend
169        ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1]))
170
171        if sys_platform in ("Linux", "Darwin"):
172            B_inv = pinv(B) if self.backend == "cpu" else jpinv(B)
173        else:
174            B_inv = pinv(B)
175
176        W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend)
177        S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend)
178
179        if sys_platform in ("Linux", "Darwin"):
180            S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat)
181        else:
182            S_inv = pinv(S_mat)
183
184        Y = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend)
185        inv = mo.rbind(
186            mo.cbind(
187                x=B_inv + mo.crossprod(x=W, y=Y, backend=self.backend),
188                y=-np.transpose(Y),
189                backend=self.backend,
190            ),
191            mo.cbind(x=-Y, y=S_inv, backend=self.backend),
192            backend=self.backend,
193        )
194
195        self.beta_ = mo.safe_sparse_dot(
196            a=inv,
197            b=mo.crossprod(x=scaled_Z, y=centered_y, backend=self.backend),
198            backend=self.backend,
199        )
200
201        return self

Fit Ridge model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
203    def predict(self, X, **kwargs):
204        """Predict test data X.
205
206        Args:
207
208            X: {array-like}, shape = [n_samples, n_features]
209                Training vectors, where n_samples is the number
210                of samples and n_features is the number of features.
211
212            **kwargs: additional parameters to be passed to
213                    self.cook_test_set
214
215        Returns:
216
217            model predictions: {array-like}
218
219        """
220
221        if len(X.shape) == 1:
222            n_features = X.shape[0]
223            new_X = mo.rbind(
224                x=X.reshape(1, n_features),
225                y=np.ones(n_features).reshape(1, n_features),
226                backend=self.backend,
227            )
228
229            return (
230                self.y_mean_
231                + mo.safe_sparse_dot(
232                    a=self.cook_test_set(new_X, **kwargs),
233                    b=self.beta_,
234                    backend=self.backend,
235                )
236            )[0]
237
238        return self.y_mean_ + mo.safe_sparse_dot(
239            a=self.cook_test_set(X, **kwargs),
240            b=self.beta_,
241            backend=self.backend,
242        )

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def score(self, X, y, scoring=None, **kwargs):
244    def score(self, X, y, scoring=None, **kwargs):
245        """ Score the model on test set features X and response y. 
246
247        Args:
248        
249            X: {array-like}, shape = [n_samples, n_features]
250                Training vectors, where n_samples is the number 
251                of samples and n_features is the number of features
252
253            y: array-like, shape = [n_samples]
254                Target values
255
256            scoring: str
257                must be in ('explained_variance', 'neg_mean_absolute_error', \
258                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
259                            'neg_median_absolute_error', 'r2')
260            
261            **kwargs: additional parameters to be passed to scoring functions
262               
263        Returns: 
264        
265            model scores: {array-like}
266            
267        """
268
269        preds = self.predict(X)
270
271        if type(preds) == tuple:  # if there are std. devs in the predictions
272            preds = preds[0]
273
274        if scoring is None:
275            scoring = "neg_mean_squared_error"
276
277        # check inputs
278        assert scoring in (
279            "explained_variance",
280            "neg_mean_absolute_error",
281            "neg_mean_squared_error",
282            "neg_mean_squared_log_error",
283            "neg_median_absolute_error",
284            "r2",
285        ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
286                           'neg_mean_squared_error', 'neg_mean_squared_log_error', \
287                           'neg_median_absolute_error', 'r2')"
288
289        scoring_options = {
290            "explained_variance": skm.explained_variance_score,
291            "neg_mean_absolute_error": skm.median_absolute_error,
292            "neg_mean_squared_error": skm.mean_squared_error,
293            "neg_mean_squared_log_error": skm.mean_squared_log_error,
294            "neg_median_absolute_error": skm.median_absolute_error,
295            "r2": skm.r2_score,
296        }
297
298        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                             'neg_mean_squared_error', 'neg_mean_squared_log_error',                             'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.ridge2.ridge2.Ridge2
lambda1
lambda2
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
class Ridge2Classifier(nnetsauce.ridge2.ridge2.Ridge2, sklearn.base.ClassifierMixin):
 18class Ridge2Classifier(Ridge2, ClassifierMixin):
 19    """Multinomial logit classification with 2 regularization parameters
 20
 21    Parameters:
 22
 23        n_hidden_features: int
 24            number of nodes in the hidden layer
 25
 26        activation_name: str
 27            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 28
 29        a: float
 30            hyperparameter for 'prelu' or 'elu' activation function
 31
 32        nodes_sim: str
 33            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 34            'uniform'
 35
 36        bias: boolean
 37            indicates if the hidden layer contains a bias term (True) or not
 38            (False)
 39
 40        dropout: float
 41            regularization parameter; (random) percentage of nodes dropped out
 42            of the training
 43
 44        direct_link: boolean
 45            indicates if the original predictors are included (True) in model's
 46            fitting or not (False)
 47
 48        n_clusters: int
 49            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 50                no clustering)
 51
 52        cluster_encode: bool
 53            defines how the variable containing clusters is treated (default is one-hot)
 54            if `False`, then labels are used, without one-hot encoding
 55
 56        type_clust: str
 57            type of clustering method: currently k-means ('kmeans') or Gaussian
 58            Mixture Model ('gmm')
 59
 60        type_scaling: a tuple of 3 strings
 61            scaling methods for inputs, hidden layer, and clustering respectively
 62            (and when relevant).
 63            Currently available: standardization ('std') or MinMax scaling ('minmax')
 64
 65        lambda1: float
 66            regularization parameter on direct link
 67
 68        lambda2: float
 69            regularization parameter on hidden layer
 70
 71        seed: int
 72            reproducibility seed for nodes_sim=='uniform'
 73
 74        backend: str
 75            "cpu" or "gpu" or "tpu"
 76
 77    Attributes:
 78
 79        beta_: {array-like}
 80            regression coefficients
 81
 82
 83    Examples:
 84
 85    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/ridge_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/ridge_classification.py)
 86
 87    ```python
 88    import nnetsauce as ns
 89    import numpy as np
 90    from sklearn.datasets import load_breast_cancer
 91    from sklearn.model_selection import train_test_split
 92    from time import time
 93
 94
 95    breast_cancer = load_breast_cancer()
 96    X = breast_cancer.data
 97    y = breast_cancer.target
 98
 99    # split data into training test and test set
100    np.random.seed(123)
101    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
102
103    # create the model with nnetsauce
104    fit_obj = ns.Ridge2Classifier(lambda1 = 6.90185578e+04,
105                                lambda2 = 3.17392781e+02,
106                                n_hidden_features=95,
107                                n_clusters=2,
108                                dropout = 3.62817383e-01,
109                                type_clust = "gmm")
110
111    # fit the model on training set
112    start = time()
113    fit_obj.fit(X_train, y_train)
114    print(f"Elapsed {time() - start}")
115
116    # get the accuracy on test set
117    start = time()
118    print(fit_obj.score(X_test, y_test))
119    print(f"Elapsed {time() - start}")
120
121    # get area under the curve on test set (auc)
122    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
123    ```
124
125
126    """
127
128    # construct the object -----
129
130    def __init__(
131        self,
132        n_hidden_features=5,
133        activation_name="relu",
134        a=0.01,
135        nodes_sim="sobol",
136        bias=True,
137        dropout=0,
138        direct_link=True,
139        n_clusters=2,
140        cluster_encode=True,
141        type_clust="kmeans",
142        type_scaling=("std", "std", "std"),
143        lambda1=0.1,
144        lambda2=0.1,
145        seed=123,
146        backend="cpu",
147    ):
148        super().__init__(
149            n_hidden_features=n_hidden_features,
150            activation_name=activation_name,
151            a=a,
152            nodes_sim=nodes_sim,
153            bias=bias,
154            dropout=dropout,
155            direct_link=direct_link,
156            n_clusters=n_clusters,
157            cluster_encode=cluster_encode,
158            type_clust=type_clust,
159            type_scaling=type_scaling,
160            lambda1=lambda1,
161            lambda2=lambda2,
162            seed=seed,
163            backend=backend,
164        )
165
166        self.type_fit = "classification"
167
168    def loglik(self, X, Y, **kwargs):
169        """Log-likelihood for training data (X, Y).
170
171        Args:
172
173            X: {array-like}, shape = [n_samples, n_features]
174                Training vectors, where n_samples is the number
175                of samples and n_features is the number of features.
176
177            Y: array-like, shape = [n_samples]
178                One-hot encode target values.
179
180            **kwargs: additional parameters to be passed to
181                    self.cook_training_set or self.obj.fit
182
183        Returns:
184
185        """
186
187        def loglik_grad_hess(Y, X, B, XB, hessian=True, **kwargs):
188            # nobs, n_classes
189            n, K = Y.shape
190
191            # total number of covariates
192            p = X.shape[1]
193
194            # initial number of covariates
195            init_p = p - self.n_hidden_features
196
197            max_double = 709.0
198            XB[XB > max_double] = max_double
199            exp_XB = np.exp(XB)
200            probs = exp_XB / exp_XB.sum(axis=1)[:, None]
201
202            # gradient -----
203            # (Y - p) -> (n, K)
204            # X -> (n, p)
205            # (K, n) %*% (n, p) -> (K, p)
206            if hessian is False:
207                grad = (
208                    -mo.safe_sparse_dot(
209                        a=(Y - probs).T, b=X, backend=self.backend
210                    )
211                    / n
212                )
213                grad += self.lambda1 * B[0:init_p, :].sum(axis=0)[:, None]
214                grad += self.lambda2 * B[init_p:p, :].sum(axis=0)[:, None]
215
216                return grad.flatten()
217
218            # hessian -----
219            if hessian is True:
220                Kp = K * p
221                hess = np.zeros((Kp, Kp), float)
222                for k1 in range(K):
223                    x_index = range(k1 * p, (k1 + 1) * p)
224                    for k2 in range(k1, K):
225                        y_index = range(k2 * p, (k2 + 1) * p)
226                        H_sub = (
227                            -mo.safe_sparse_dot(
228                                a=X.T,
229                                b=(probs[:, k1] * probs[:, k2])[:, None] * X,
230                                backend=self.backend,
231                            )
232                            / n
233                        )  # do not store
234                        hess[np.ix_(x_index, y_index)] = hess[
235                            np.ix_(y_index, x_index)
236                        ] = H_sub
237
238                return hess + (self.lambda1 + self.lambda2) * np.identity(Kp)
239
240        # total number of covariates
241        p = X.shape[1]
242
243        # initial number of covariates
244        init_p = p - self.n_hidden_features
245
246        # log-likelihood (1st return)
247        def loglik_func(x):
248            # (p, K)
249            B = x.reshape(Y.shape[1], p).T
250
251            # (n, K)
252            XB = mo.safe_sparse_dot(X, B, backend=self.backend)
253
254            res = -(np.sum(Y * XB, axis=1) - logsumexp(XB)).mean()
255
256            res += (
257                0.5
258                * self.lambda1
259                * mo.squared_norm(B[0:init_p, :], backend=self.backend)
260            )
261            res += (
262                0.5
263                * self.lambda2
264                * mo.squared_norm(B[init_p:p, :], backend=self.backend)
265            )
266
267            return res
268
269        # gradient of log-likelihood
270        def grad_func(x):
271            # (p, K)
272            B = x.reshape(Y.shape[1], p).T
273
274            return loglik_grad_hess(
275                Y=Y,
276                X=X,
277                B=B,
278                XB=mo.safe_sparse_dot(X, B, backend=self.backend),
279                hessian=False,
280                **kwargs
281            )
282
283        # hessian of log-likelihood
284        def hessian_func(x):
285            # (p, K)
286            B = x.reshape(Y.shape[1], p).T
287
288            return loglik_grad_hess(
289                Y=Y,
290                X=X,
291                B=B,
292                XB=mo.safe_sparse_dot(X, B, backend=self.backend),
293                hessian=True,
294                **kwargs
295            )
296
297        return loglik_func, grad_func, hessian_func
298
299    # newton-cg
300    # L-BFGS-B
301    def fit(self, X, y, solver="L-BFGS-B", **kwargs):
302        """Fit Ridge model to training data (X, y).
303
304        for beta: regression coeffs (beta11, ..., beta1p, ..., betaK1, ..., betaKp)
305        for K classes and p covariates.
306
307        Args:
308
309            X: {array-like}, shape = [n_samples, n_features]
310                Training vectors, where n_samples is the number
311                of samples and n_features is the number of features.
312
313            y: array-like, shape = [n_samples]
314                Target values.
315
316            **kwargs: additional parameters to be passed to
317                    self.cook_training_set or self.obj.fit
318
319        Returns:
320
321            self: object
322
323        """
324
325        assert mx.is_factor(y), "y must contain only integers"
326
327        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
328
329        self.n_classes = len(np.unique(y))
330
331        Y = mo.one_hot_encode2(output_y, self.n_classes)
332
333        # optimize for beta, minimize self.loglik (maximize loglik) -----
334        loglik_func, grad_func, hessian_func = self.loglik(X=scaled_Z, Y=Y)
335
336        if solver == "L-BFGS-B":
337            self.beta_ = minimize(
338                fun=loglik_func,
339                x0=np.zeros(scaled_Z.shape[1] * self.n_classes),
340                jac=grad_func,
341                method=solver,
342            ).x
343
344        if solver in ("Newton-CG", "trust-ncg"):
345            self.beta_ = minimize(
346                fun=loglik_func,
347                x0=np.zeros(scaled_Z.shape[1] * self.n_classes),
348                jac=grad_func,
349                hess=hessian_func,
350                method=solver,
351            ).x
352
353        return self
354
355    def predict(self, X, **kwargs):
356        """Predict test data X.
357
358        Args:
359
360            X: {array-like}, shape = [n_samples, n_features]
361                Training vectors, where n_samples is the number
362                of samples and n_features is the number of features.
363
364            **kwargs: additional parameters to be passed to
365                    self.cook_test_set
366
367        Returns:
368
369            model predictions: {array-like}
370        """
371
372        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
373
374    def predict_proba(self, X, **kwargs):
375        """Predict probabilities for test data X.
376
377        Args:
378
379            X: {array-like}, shape = [n_samples, n_features]
380                Training vectors, where n_samples is the number
381                of samples and n_features is the number of features.
382
383            **kwargs: additional parameters to be passed to
384                    self.cook_test_set
385
386        Returns:
387
388            probability estimates for test data: {array-like}
389
390        """
391        if len(X.shape) == 1:
392            n_features = X.shape[0]
393            new_X = mo.rbind(
394                X.reshape(1, n_features),
395                np.ones(n_features).reshape(1, n_features),
396            )
397
398            Z = self.cook_test_set(new_X, **kwargs)
399
400        else:
401            Z = self.cook_test_set(X, **kwargs)
402
403        ZB = mo.safe_sparse_dot(
404            a=Z,
405            b=self.beta_.reshape(
406                self.n_classes,
407                X.shape[1] + self.n_hidden_features + self.n_clusters,
408            ).T,
409            backend=self.backend,
410        )
411
412        exp_ZB = np.exp(ZB)
413
414        return exp_ZB / exp_ZB.sum(axis=1)[:, None]
415
416    def score(self, X, y, scoring=None, **kwargs):
417        """ Score the model on test set features X and response y. 
418
419        Args:
420        
421            X: {array-like}, shape = [n_samples, n_features]
422                Training vectors, where n_samples is the number 
423                of samples and n_features is the number of features
424
425            y: array-like, shape = [n_samples]
426                Target values
427
428            scoring: str
429                must be in ('explained_variance', 'neg_mean_absolute_error', \
430                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
431                            'neg_median_absolute_error', 'r2')
432            
433            **kwargs: additional parameters to be passed to scoring functions
434               
435        Returns: 
436        
437            model scores: {array-like}
438
439        """
440
441        preds = self.predict(X)
442
443        if scoring is None:
444            scoring = "accuracy"
445
446        # check inputs
447        assert scoring in (
448            "accuracy",
449            "average_precision",
450            "brier_score_loss",
451            "f1",
452            "f1_micro",
453            "f1_macro",
454            "f1_weighted",
455            "f1_samples",
456            "neg_log_loss",
457            "precision",
458            "recall",
459            "roc_auc",
460        ), "'scoring' should be in ('accuracy', 'average_precision', \
461                           'brier_score_loss', 'f1', 'f1_micro', \
462                           'f1_macro', 'f1_weighted',  'f1_samples', \
463                           'neg_log_loss', 'precision', 'recall', \
464                           'roc_auc')"
465
466        scoring_options = {
467            "accuracy": skm2.accuracy_score,
468            "average_precision": skm2.average_precision_score,
469            "brier_score_loss": skm2.brier_score_loss,
470            "f1": skm2.f1_score,
471            "f1_micro": skm2.f1_score,
472            "f1_macro": skm2.f1_score,
473            "f1_weighted": skm2.f1_score,
474            "f1_samples": skm2.f1_score,
475            "neg_log_loss": skm2.log_loss,
476            "precision": skm2.precision_score,
477            "recall": skm2.recall_score,
478            "roc_auc": skm2.roc_auc_score,
479        }
480
481        return scoring_options[scoring](y, preds, **kwargs)

Multinomial logit classification with 2 regularization parameters

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

direct_link: boolean
    indicates if the original predictors are included (True) in model's
    fitting or not (False)

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

lambda1: float
    regularization parameter on direct link

lambda2: float
    regularization parameter on hidden layer

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

beta_: {array-like}
    regression coefficients

Examples:

See also https://github.com/Techtonique/nnetsauce/blob/master/examples/ridge_classification.py

import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from time import time


breast_cancer = load_breast_cancer()
X = breast_cancer.data
y = breast_cancer.target

# split data into training test and test set
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# create the model with nnetsauce
fit_obj = ns.Ridge2Classifier(lambda1 = 6.90185578e+04,
                            lambda2 = 3.17392781e+02,
                            n_hidden_features=95,
                            n_clusters=2,
                            dropout = 3.62817383e-01,
                            type_clust = "gmm")

# fit the model on training set
start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

# get the accuracy on test set
start = time()
print(fit_obj.score(X_test, y_test))
print(f"Elapsed {time() - start}")

# get area under the curve on test set (auc)
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
Ridge2Classifier( n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, direct_link=True, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), lambda1=0.1, lambda2=0.1, seed=123, backend='cpu')
130    def __init__(
131        self,
132        n_hidden_features=5,
133        activation_name="relu",
134        a=0.01,
135        nodes_sim="sobol",
136        bias=True,
137        dropout=0,
138        direct_link=True,
139        n_clusters=2,
140        cluster_encode=True,
141        type_clust="kmeans",
142        type_scaling=("std", "std", "std"),
143        lambda1=0.1,
144        lambda2=0.1,
145        seed=123,
146        backend="cpu",
147    ):
148        super().__init__(
149            n_hidden_features=n_hidden_features,
150            activation_name=activation_name,
151            a=a,
152            nodes_sim=nodes_sim,
153            bias=bias,
154            dropout=dropout,
155            direct_link=direct_link,
156            n_clusters=n_clusters,
157            cluster_encode=cluster_encode,
158            type_clust=type_clust,
159            type_scaling=type_scaling,
160            lambda1=lambda1,
161            lambda2=lambda2,
162            seed=seed,
163            backend=backend,
164        )
165
166        self.type_fit = "classification"
type_fit
def loglik(self, X, Y, **kwargs):
168    def loglik(self, X, Y, **kwargs):
169        """Log-likelihood for training data (X, Y).
170
171        Args:
172
173            X: {array-like}, shape = [n_samples, n_features]
174                Training vectors, where n_samples is the number
175                of samples and n_features is the number of features.
176
177            Y: array-like, shape = [n_samples]
178                One-hot encode target values.
179
180            **kwargs: additional parameters to be passed to
181                    self.cook_training_set or self.obj.fit
182
183        Returns:
184
185        """
186
187        def loglik_grad_hess(Y, X, B, XB, hessian=True, **kwargs):
188            # nobs, n_classes
189            n, K = Y.shape
190
191            # total number of covariates
192            p = X.shape[1]
193
194            # initial number of covariates
195            init_p = p - self.n_hidden_features
196
197            max_double = 709.0
198            XB[XB > max_double] = max_double
199            exp_XB = np.exp(XB)
200            probs = exp_XB / exp_XB.sum(axis=1)[:, None]
201
202            # gradient -----
203            # (Y - p) -> (n, K)
204            # X -> (n, p)
205            # (K, n) %*% (n, p) -> (K, p)
206            if hessian is False:
207                grad = (
208                    -mo.safe_sparse_dot(
209                        a=(Y - probs).T, b=X, backend=self.backend
210                    )
211                    / n
212                )
213                grad += self.lambda1 * B[0:init_p, :].sum(axis=0)[:, None]
214                grad += self.lambda2 * B[init_p:p, :].sum(axis=0)[:, None]
215
216                return grad.flatten()
217
218            # hessian -----
219            if hessian is True:
220                Kp = K * p
221                hess = np.zeros((Kp, Kp), float)
222                for k1 in range(K):
223                    x_index = range(k1 * p, (k1 + 1) * p)
224                    for k2 in range(k1, K):
225                        y_index = range(k2 * p, (k2 + 1) * p)
226                        H_sub = (
227                            -mo.safe_sparse_dot(
228                                a=X.T,
229                                b=(probs[:, k1] * probs[:, k2])[:, None] * X,
230                                backend=self.backend,
231                            )
232                            / n
233                        )  # do not store
234                        hess[np.ix_(x_index, y_index)] = hess[
235                            np.ix_(y_index, x_index)
236                        ] = H_sub
237
238                return hess + (self.lambda1 + self.lambda2) * np.identity(Kp)
239
240        # total number of covariates
241        p = X.shape[1]
242
243        # initial number of covariates
244        init_p = p - self.n_hidden_features
245
246        # log-likelihood (1st return)
247        def loglik_func(x):
248            # (p, K)
249            B = x.reshape(Y.shape[1], p).T
250
251            # (n, K)
252            XB = mo.safe_sparse_dot(X, B, backend=self.backend)
253
254            res = -(np.sum(Y * XB, axis=1) - logsumexp(XB)).mean()
255
256            res += (
257                0.5
258                * self.lambda1
259                * mo.squared_norm(B[0:init_p, :], backend=self.backend)
260            )
261            res += (
262                0.5
263                * self.lambda2
264                * mo.squared_norm(B[init_p:p, :], backend=self.backend)
265            )
266
267            return res
268
269        # gradient of log-likelihood
270        def grad_func(x):
271            # (p, K)
272            B = x.reshape(Y.shape[1], p).T
273
274            return loglik_grad_hess(
275                Y=Y,
276                X=X,
277                B=B,
278                XB=mo.safe_sparse_dot(X, B, backend=self.backend),
279                hessian=False,
280                **kwargs
281            )
282
283        # hessian of log-likelihood
284        def hessian_func(x):
285            # (p, K)
286            B = x.reshape(Y.shape[1], p).T
287
288            return loglik_grad_hess(
289                Y=Y,
290                X=X,
291                B=B,
292                XB=mo.safe_sparse_dot(X, B, backend=self.backend),
293                hessian=True,
294                **kwargs
295            )
296
297        return loglik_func, grad_func, hessian_func

Log-likelihood for training data (X, Y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

Y: array-like, shape = [n_samples]
    One-hot encode target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

def fit(self, X, y, solver='L-BFGS-B', **kwargs):
301    def fit(self, X, y, solver="L-BFGS-B", **kwargs):
302        """Fit Ridge model to training data (X, y).
303
304        for beta: regression coeffs (beta11, ..., beta1p, ..., betaK1, ..., betaKp)
305        for K classes and p covariates.
306
307        Args:
308
309            X: {array-like}, shape = [n_samples, n_features]
310                Training vectors, where n_samples is the number
311                of samples and n_features is the number of features.
312
313            y: array-like, shape = [n_samples]
314                Target values.
315
316            **kwargs: additional parameters to be passed to
317                    self.cook_training_set or self.obj.fit
318
319        Returns:
320
321            self: object
322
323        """
324
325        assert mx.is_factor(y), "y must contain only integers"
326
327        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
328
329        self.n_classes = len(np.unique(y))
330
331        Y = mo.one_hot_encode2(output_y, self.n_classes)
332
333        # optimize for beta, minimize self.loglik (maximize loglik) -----
334        loglik_func, grad_func, hessian_func = self.loglik(X=scaled_Z, Y=Y)
335
336        if solver == "L-BFGS-B":
337            self.beta_ = minimize(
338                fun=loglik_func,
339                x0=np.zeros(scaled_Z.shape[1] * self.n_classes),
340                jac=grad_func,
341                method=solver,
342            ).x
343
344        if solver in ("Newton-CG", "trust-ncg"):
345            self.beta_ = minimize(
346                fun=loglik_func,
347                x0=np.zeros(scaled_Z.shape[1] * self.n_classes),
348                jac=grad_func,
349                hess=hessian_func,
350                method=solver,
351            ).x
352
353        return self

Fit Ridge model to training data (X, y).

for beta: regression coeffs (beta11, ..., beta1p, ..., betaK1, ..., betaKp) for K classes and p covariates.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
355    def predict(self, X, **kwargs):
356        """Predict test data X.
357
358        Args:
359
360            X: {array-like}, shape = [n_samples, n_features]
361                Training vectors, where n_samples is the number
362                of samples and n_features is the number of features.
363
364            **kwargs: additional parameters to be passed to
365                    self.cook_test_set
366
367        Returns:
368
369            model predictions: {array-like}
370        """
371
372        return np.argmax(self.predict_proba(X, **kwargs), axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
374    def predict_proba(self, X, **kwargs):
375        """Predict probabilities for test data X.
376
377        Args:
378
379            X: {array-like}, shape = [n_samples, n_features]
380                Training vectors, where n_samples is the number
381                of samples and n_features is the number of features.
382
383            **kwargs: additional parameters to be passed to
384                    self.cook_test_set
385
386        Returns:
387
388            probability estimates for test data: {array-like}
389
390        """
391        if len(X.shape) == 1:
392            n_features = X.shape[0]
393            new_X = mo.rbind(
394                X.reshape(1, n_features),
395                np.ones(n_features).reshape(1, n_features),
396            )
397
398            Z = self.cook_test_set(new_X, **kwargs)
399
400        else:
401            Z = self.cook_test_set(X, **kwargs)
402
403        ZB = mo.safe_sparse_dot(
404            a=Z,
405            b=self.beta_.reshape(
406                self.n_classes,
407                X.shape[1] + self.n_hidden_features + self.n_clusters,
408            ).T,
409            backend=self.backend,
410        )
411
412        exp_ZB = np.exp(ZB)
413
414        return exp_ZB / exp_ZB.sum(axis=1)[:, None]

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
def score(self, X, y, scoring=None, **kwargs):
416    def score(self, X, y, scoring=None, **kwargs):
417        """ Score the model on test set features X and response y. 
418
419        Args:
420        
421            X: {array-like}, shape = [n_samples, n_features]
422                Training vectors, where n_samples is the number 
423                of samples and n_features is the number of features
424
425            y: array-like, shape = [n_samples]
426                Target values
427
428            scoring: str
429                must be in ('explained_variance', 'neg_mean_absolute_error', \
430                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
431                            'neg_median_absolute_error', 'r2')
432            
433            **kwargs: additional parameters to be passed to scoring functions
434               
435        Returns: 
436        
437            model scores: {array-like}
438
439        """
440
441        preds = self.predict(X)
442
443        if scoring is None:
444            scoring = "accuracy"
445
446        # check inputs
447        assert scoring in (
448            "accuracy",
449            "average_precision",
450            "brier_score_loss",
451            "f1",
452            "f1_micro",
453            "f1_macro",
454            "f1_weighted",
455            "f1_samples",
456            "neg_log_loss",
457            "precision",
458            "recall",
459            "roc_auc",
460        ), "'scoring' should be in ('accuracy', 'average_precision', \
461                           'brier_score_loss', 'f1', 'f1_micro', \
462                           'f1_macro', 'f1_weighted',  'f1_samples', \
463                           'neg_log_loss', 'precision', 'recall', \
464                           'roc_auc')"
465
466        scoring_options = {
467            "accuracy": skm2.accuracy_score,
468            "average_precision": skm2.average_precision_score,
469            "brier_score_loss": skm2.brier_score_loss,
470            "f1": skm2.f1_score,
471            "f1_micro": skm2.f1_score,
472            "f1_macro": skm2.f1_score,
473            "f1_weighted": skm2.f1_score,
474            "f1_samples": skm2.f1_score,
475            "neg_log_loss": skm2.log_loss,
476            "precision": skm2.precision_score,
477            "recall": skm2.recall_score,
478            "roc_auc": skm2.roc_auc_score,
479        }
480
481        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set features X and response y.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number 
    of samples and n_features is the number of features

y: array-like, shape = [n_samples]
    Target values

scoring: str
    must be in ('explained_variance', 'neg_mean_absolute_error',                             'neg_mean_squared_error', 'neg_mean_squared_log_error',                             'neg_median_absolute_error', 'r2')

**kwargs: additional parameters to be passed to scoring functions

Returns:

model scores: {array-like}
def set_fit_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.ridge2.ridge2.Ridge2
lambda1
lambda2
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
class Ridge2MultitaskClassifier(nnetsauce.ridge2.ridge2.Ridge2, sklearn.base.ClassifierMixin):
 21class Ridge2MultitaskClassifier(Ridge2, ClassifierMixin):
 22    """Multitask Ridge classification with 2 regularization parameters
 23
 24    Parameters:
 25
 26        n_hidden_features: int
 27            number of nodes in the hidden layer
 28
 29        activation_name: str
 30            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 31
 32        a: float
 33            hyperparameter for 'prelu' or 'elu' activation function
 34
 35        nodes_sim: str
 36            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 37            'uniform'
 38
 39        bias: boolean
 40            indicates if the hidden layer contains a bias term (True) or not
 41            (False)
 42
 43        dropout: float
 44            regularization parameter; (random) percentage of nodes dropped out
 45            of the training
 46
 47        n_clusters: int
 48            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 49                no clustering)
 50
 51        cluster_encode: bool
 52            defines how the variable containing clusters is treated (default is one-hot)
 53            if `False`, then labels are used, without one-hot encoding
 54
 55        type_clust: str
 56            type of clustering method: currently k-means ('kmeans') or Gaussian
 57            Mixture Model ('gmm')
 58
 59        type_scaling: a tuple of 3 strings
 60            scaling methods for inputs, hidden layer, and clustering respectively
 61            (and when relevant).
 62            Currently available: standardization ('std') or MinMax scaling ('minmax')
 63
 64        lambda1: float
 65            regularization parameter on direct link
 66
 67        lambda2: float
 68            regularization parameter on hidden layer
 69
 70        seed: int
 71            reproducibility seed for nodes_sim=='uniform'
 72
 73        backend: str
 74            "cpu" or "gpu" or "tpu"
 75
 76    Attributes:
 77
 78        beta_: {array-like}
 79            regression coefficients
 80
 81    Examples:
 82
 83    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/ridgemtask_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/ridgemtask_classification.py)
 84
 85    ```python
 86    import nnetsauce as ns
 87    import numpy as np
 88    from sklearn.datasets import load_breast_cancer
 89    from sklearn.model_selection import train_test_split
 90    from sklearn import metrics
 91    from time import time
 92
 93    breast_cancer = load_breast_cancer()
 94    Z = breast_cancer.data
 95    t = breast_cancer.target
 96    np.random.seed(123)
 97    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
 98
 99    fit_obj = ns.Ridge2MultitaskClassifier(n_hidden_features=int(9.83730469e+01),
100                                    dropout=4.31054687e-01,
101                                    n_clusters=int(1.71484375e+00),
102                                    lambda1=1.24023438e+01, lambda2=7.30263672e+03)
103
104    start = time()
105    fit_obj.fit(X_train, y_train)
106    print(f"Elapsed {time() - start}")
107
108    print(fit_obj.score(X_test, y_test))
109    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
110
111    start = time()
112    preds = fit_obj.predict(X_test)
113    print(f"Elapsed {time() - start}")
114    print(metrics.classification_report(preds, y_test))
115    ```
116
117    """
118
119    # construct the object -----
120
121    def __init__(
122        self,
123        n_hidden_features=5,
124        activation_name="relu",
125        a=0.01,
126        nodes_sim="sobol",
127        bias=True,
128        dropout=0,
129        n_clusters=2,
130        cluster_encode=True,
131        type_clust="kmeans",
132        type_scaling=("std", "std", "std"),
133        lambda1=0.1,
134        lambda2=0.1,
135        seed=123,
136        backend="cpu",
137    ):
138        super().__init__(
139            n_hidden_features=n_hidden_features,
140            activation_name=activation_name,
141            a=a,
142            nodes_sim=nodes_sim,
143            bias=bias,
144            dropout=dropout,
145            n_clusters=n_clusters,
146            cluster_encode=cluster_encode,
147            type_clust=type_clust,
148            type_scaling=type_scaling,
149            lambda1=lambda1,
150            lambda2=lambda2,
151            seed=seed,
152            backend=backend,
153        )
154
155        self.type_fit = "regression"
156
157    def fit(self, X, y, **kwargs):
158        """Fit Ridge model to training data (X, y).
159
160        Args:
161
162            X: {array-like}, shape = [n_samples, n_features]
163                Training vectors, where n_samples is the number
164                of samples and n_features is the number of features.
165
166            y: array-like, shape = [n_samples]
167                Target values.
168
169            **kwargs: additional parameters to be passed to
170                    self.cook_training_set or self.obj.fit
171
172        Returns:
173
174            self: object
175
176        """
177
178        sys_platform = platform.system()
179
180        assert mx.is_factor(y), "y must contain only integers"
181
182        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
183
184        n_X, p_X = X.shape
185        n_Z, p_Z = scaled_Z.shape
186
187        self.n_classes = len(np.unique(y))
188
189        # multitask response
190        Y = mo.one_hot_encode2(output_y, self.n_classes)
191
192        if self.n_clusters > 0:
193            if self.encode_clusters == True:
194                n_features = p_X + self.n_clusters
195            else:
196                n_features = p_X + 1
197        else:
198            n_features = p_X
199
200        X_ = scaled_Z[:, 0:n_features]
201        Phi_X_ = scaled_Z[:, n_features:p_Z]
202
203        B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag(
204            np.repeat(1, X_.shape[1])
205        )
206        C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend)
207        D = mo.crossprod(
208            x=Phi_X_, backend=self.backend
209        ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1]))
210
211        if sys_platform in ("Linux", "Darwin"):
212            B_inv = pinv(B) if self.backend == "cpu" else jpinv(B)
213        else:
214            B_inv = pinv(B)
215
216        W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend)
217        S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend)
218
219        if sys_platform in ("Linux", "Darwin"):
220            S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat)
221        else:
222            S_inv = pinv(S_mat)
223
224        Y2 = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend)
225        inv = mo.rbind(
226            mo.cbind(
227                x=B_inv + mo.crossprod(x=W, y=Y2, backend=self.backend),
228                y=-np.transpose(Y2),
229                backend=self.backend,
230            ),
231            mo.cbind(x=-Y2, y=S_inv, backend=self.backend),
232            backend=self.backend,
233        )
234
235        self.beta_ = mo.safe_sparse_dot(
236            a=inv,
237            b=mo.crossprod(x=scaled_Z, y=Y, backend=self.backend),
238            backend=self.backend,
239        )
240
241        return self
242
243    def predict(self, X, **kwargs):
244        """Predict test data X.
245
246        Args:
247
248            X: {array-like}, shape = [n_samples, n_features]
249                Training vectors, where n_samples is the number
250                of samples and n_features is the number of features.
251
252            **kwargs: additional parameters to be passed to
253                    self.cook_test_set
254
255        Returns:
256
257            model predictions: {array-like}
258
259        """
260
261        return np.argmax(self.predict_proba(X, **kwargs), axis=1)
262
263    def predict_proba(self, X, **kwargs):
264        """Predict probabilities for test data X.
265
266        Args:
267
268            X: {array-like}, shape = [n_samples, n_features]
269                Training vectors, where n_samples is the number
270                of samples and n_features is the number of features.
271
272            **kwargs: additional parameters to be passed to
273                    self.cook_test_set
274
275        Returns:
276
277            probability estimates for test data: {array-like}
278
279        """
280
281        if len(X.shape) == 1:
282            n_features = X.shape[0]
283            new_X = mo.rbind(
284                x=X.reshape(1, n_features),
285                y=np.ones(n_features).reshape(1, n_features),
286                backend=self.backend,
287            )
288
289            Z = self.cook_test_set(new_X, **kwargs)
290
291        else:
292            Z = self.cook_test_set(X, **kwargs)
293
294        ZB = mo.safe_sparse_dot(a=Z, b=self.beta_, backend=self.backend)
295
296        exp_ZB = np.exp(ZB)
297
298        return exp_ZB / exp_ZB.sum(axis=1)[:, None]
299
300    def score(self, X, y, scoring=None, **kwargs):
301        """Score the model on test set covariates X and response y."""
302
303        preds = self.predict(X)
304
305        if scoring is None:
306            scoring = "accuracy"
307
308        # check inputs
309        assert scoring in (
310            "accuracy",
311            "average_precision",
312            "brier_score_loss",
313            "f1",
314            "f1_micro",
315            "f1_macro",
316            "f1_weighted",
317            "f1_samples",
318            "neg_log_loss",
319            "precision",
320            "recall",
321            "roc_auc",
322        ), "'scoring' should be in ('accuracy', 'average_precision', \
323                           'brier_score_loss', 'f1', 'f1_micro', \
324                           'f1_macro', 'f1_weighted',  'f1_samples', \
325                           'neg_log_loss', 'precision', 'recall', \
326                           'roc_auc')"
327
328        scoring_options = {
329            "accuracy": skm2.accuracy_score,
330            "average_precision": skm2.average_precision_score,
331            "brier_score_loss": skm2.brier_score_loss,
332            "f1": skm2.f1_score,
333            "f1_micro": skm2.f1_score,
334            "f1_macro": skm2.f1_score,
335            "f1_weighted": skm2.f1_score,
336            "f1_samples": skm2.f1_score,
337            "neg_log_loss": skm2.log_loss,
338            "precision": skm2.precision_score,
339            "recall": skm2.recall_score,
340            "roc_auc": skm2.roc_auc_score,
341        }
342
343        return scoring_options[scoring](y, preds, **kwargs)

Multitask Ridge classification with 2 regularization parameters

Parameters:

n_hidden_features: int
    number of nodes in the hidden layer

activation_name: str
    activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'

a: float
    hyperparameter for 'prelu' or 'elu' activation function

nodes_sim: str
    type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
    'uniform'

bias: boolean
    indicates if the hidden layer contains a bias term (True) or not
    (False)

dropout: float
    regularization parameter; (random) percentage of nodes dropped out
    of the training

n_clusters: int
    number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
        no clustering)

cluster_encode: bool
    defines how the variable containing clusters is treated (default is one-hot)
    if `False`, then labels are used, without one-hot encoding

type_clust: str
    type of clustering method: currently k-means ('kmeans') or Gaussian
    Mixture Model ('gmm')

type_scaling: a tuple of 3 strings
    scaling methods for inputs, hidden layer, and clustering respectively
    (and when relevant).
    Currently available: standardization ('std') or MinMax scaling ('minmax')

lambda1: float
    regularization parameter on direct link

lambda2: float
    regularization parameter on hidden layer

seed: int
    reproducibility seed for nodes_sim=='uniform'

backend: str
    "cpu" or "gpu" or "tpu"

Attributes:

beta_: {array-like}
    regression coefficients

Examples:

See also https://github.com/Techtonique/nnetsauce/blob/master/examples/ridgemtask_classification.py

import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time

breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)

fit_obj = ns.Ridge2MultitaskClassifier(n_hidden_features=int(9.83730469e+01),
                                dropout=4.31054687e-01,
                                n_clusters=int(1.71484375e+00),
                                lambda1=1.24023438e+01, lambda2=7.30263672e+03)

start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")

print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))

start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
Ridge2MultitaskClassifier( n_hidden_features=5, activation_name='relu', a=0.01, nodes_sim='sobol', bias=True, dropout=0, n_clusters=2, cluster_encode=True, type_clust='kmeans', type_scaling=('std', 'std', 'std'), lambda1=0.1, lambda2=0.1, seed=123, backend='cpu')
121    def __init__(
122        self,
123        n_hidden_features=5,
124        activation_name="relu",
125        a=0.01,
126        nodes_sim="sobol",
127        bias=True,
128        dropout=0,
129        n_clusters=2,
130        cluster_encode=True,
131        type_clust="kmeans",
132        type_scaling=("std", "std", "std"),
133        lambda1=0.1,
134        lambda2=0.1,
135        seed=123,
136        backend="cpu",
137    ):
138        super().__init__(
139            n_hidden_features=n_hidden_features,
140            activation_name=activation_name,
141            a=a,
142            nodes_sim=nodes_sim,
143            bias=bias,
144            dropout=dropout,
145            n_clusters=n_clusters,
146            cluster_encode=cluster_encode,
147            type_clust=type_clust,
148            type_scaling=type_scaling,
149            lambda1=lambda1,
150            lambda2=lambda2,
151            seed=seed,
152            backend=backend,
153        )
154
155        self.type_fit = "regression"
type_fit
def fit(self, X, y, **kwargs):
157    def fit(self, X, y, **kwargs):
158        """Fit Ridge model to training data (X, y).
159
160        Args:
161
162            X: {array-like}, shape = [n_samples, n_features]
163                Training vectors, where n_samples is the number
164                of samples and n_features is the number of features.
165
166            y: array-like, shape = [n_samples]
167                Target values.
168
169            **kwargs: additional parameters to be passed to
170                    self.cook_training_set or self.obj.fit
171
172        Returns:
173
174            self: object
175
176        """
177
178        sys_platform = platform.system()
179
180        assert mx.is_factor(y), "y must contain only integers"
181
182        output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs)
183
184        n_X, p_X = X.shape
185        n_Z, p_Z = scaled_Z.shape
186
187        self.n_classes = len(np.unique(y))
188
189        # multitask response
190        Y = mo.one_hot_encode2(output_y, self.n_classes)
191
192        if self.n_clusters > 0:
193            if self.encode_clusters == True:
194                n_features = p_X + self.n_clusters
195            else:
196                n_features = p_X + 1
197        else:
198            n_features = p_X
199
200        X_ = scaled_Z[:, 0:n_features]
201        Phi_X_ = scaled_Z[:, n_features:p_Z]
202
203        B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag(
204            np.repeat(1, X_.shape[1])
205        )
206        C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend)
207        D = mo.crossprod(
208            x=Phi_X_, backend=self.backend
209        ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1]))
210
211        if sys_platform in ("Linux", "Darwin"):
212            B_inv = pinv(B) if self.backend == "cpu" else jpinv(B)
213        else:
214            B_inv = pinv(B)
215
216        W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend)
217        S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend)
218
219        if sys_platform in ("Linux", "Darwin"):
220            S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat)
221        else:
222            S_inv = pinv(S_mat)
223
224        Y2 = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend)
225        inv = mo.rbind(
226            mo.cbind(
227                x=B_inv + mo.crossprod(x=W, y=Y2, backend=self.backend),
228                y=-np.transpose(Y2),
229                backend=self.backend,
230            ),
231            mo.cbind(x=-Y2, y=S_inv, backend=self.backend),
232            backend=self.backend,
233        )
234
235        self.beta_ = mo.safe_sparse_dot(
236            a=inv,
237            b=mo.crossprod(x=scaled_Z, y=Y, backend=self.backend),
238            backend=self.backend,
239        )
240
241        return self

Fit Ridge model to training data (X, y).

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

y: array-like, shape = [n_samples]
    Target values.

**kwargs: additional parameters to be passed to
        self.cook_training_set or self.obj.fit

Returns:

self: object
def predict(self, X, **kwargs):
243    def predict(self, X, **kwargs):
244        """Predict test data X.
245
246        Args:
247
248            X: {array-like}, shape = [n_samples, n_features]
249                Training vectors, where n_samples is the number
250                of samples and n_features is the number of features.
251
252            **kwargs: additional parameters to be passed to
253                    self.cook_test_set
254
255        Returns:
256
257            model predictions: {array-like}
258
259        """
260
261        return np.argmax(self.predict_proba(X, **kwargs), axis=1)

Predict test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

model predictions: {array-like}
def predict_proba(self, X, **kwargs):
263    def predict_proba(self, X, **kwargs):
264        """Predict probabilities for test data X.
265
266        Args:
267
268            X: {array-like}, shape = [n_samples, n_features]
269                Training vectors, where n_samples is the number
270                of samples and n_features is the number of features.
271
272            **kwargs: additional parameters to be passed to
273                    self.cook_test_set
274
275        Returns:
276
277            probability estimates for test data: {array-like}
278
279        """
280
281        if len(X.shape) == 1:
282            n_features = X.shape[0]
283            new_X = mo.rbind(
284                x=X.reshape(1, n_features),
285                y=np.ones(n_features).reshape(1, n_features),
286                backend=self.backend,
287            )
288
289            Z = self.cook_test_set(new_X, **kwargs)
290
291        else:
292            Z = self.cook_test_set(X, **kwargs)
293
294        ZB = mo.safe_sparse_dot(a=Z, b=self.beta_, backend=self.backend)
295
296        exp_ZB = np.exp(ZB)
297
298        return exp_ZB / exp_ZB.sum(axis=1)[:, None]

Predict probabilities for test data X.

Args:

X: {array-like}, shape = [n_samples, n_features]
    Training vectors, where n_samples is the number
    of samples and n_features is the number of features.

**kwargs: additional parameters to be passed to
        self.cook_test_set

Returns:

probability estimates for test data: {array-like}
def score(self, X, y, scoring=None, **kwargs):
300    def score(self, X, y, scoring=None, **kwargs):
301        """Score the model on test set covariates X and response y."""
302
303        preds = self.predict(X)
304
305        if scoring is None:
306            scoring = "accuracy"
307
308        # check inputs
309        assert scoring in (
310            "accuracy",
311            "average_precision",
312            "brier_score_loss",
313            "f1",
314            "f1_micro",
315            "f1_macro",
316            "f1_weighted",
317            "f1_samples",
318            "neg_log_loss",
319            "precision",
320            "recall",
321            "roc_auc",
322        ), "'scoring' should be in ('accuracy', 'average_precision', \
323                           'brier_score_loss', 'f1', 'f1_micro', \
324                           'f1_macro', 'f1_weighted',  'f1_samples', \
325                           'neg_log_loss', 'precision', 'recall', \
326                           'roc_auc')"
327
328        scoring_options = {
329            "accuracy": skm2.accuracy_score,
330            "average_precision": skm2.average_precision_score,
331            "brier_score_loss": skm2.brier_score_loss,
332            "f1": skm2.f1_score,
333            "f1_micro": skm2.f1_score,
334            "f1_macro": skm2.f1_score,
335            "f1_weighted": skm2.f1_score,
336            "f1_samples": skm2.f1_score,
337            "neg_log_loss": skm2.log_loss,
338            "precision": skm2.precision_score,
339            "recall": skm2.recall_score,
340            "roc_auc": skm2.roc_auc_score,
341        }
342
343        return scoring_options[scoring](y, preds, **kwargs)

Score the model on test set covariates X and response y.

def set_score_request(unknown):

A descriptor for request methods.

New in version 1.3.

Parameters

name : str The name of the method for which the request function should be created, e.g. "fit" would create a set_fit_request function.

keys : list of str A list of strings which are accepted parameters by the created function, e.g. ["sample_weight"] if the corresponding method accepts it as a metadata.

validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.

Notes

This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.

References

Inherited Members
nnetsauce.ridge2.ridge2.Ridge2
lambda1
lambda2
Base
n_hidden_features
activation_name
a
nodes_sim
bias
seed
backend
dropout
cluster_encode
type_clust
type_scaling
col_sample
row_sample
n_clusters
subsampler_
index_col_
index_row_
clustering_obj_
clustering_scaler_
nn_scaler_
scaler_
encoder_
W_
X_
y_
y_mean_
beta_
activation_func
encode_clusters
create_layer
cook_training_set
cook_test_set
sklearn.base.BaseEstimator
get_params
set_params
sklearn.utils._metadata_requests._MetadataRequester
get_metadata_routing
class SubSampler:
 5class SubSampler:
 6    """Subsampling class.
 7
 8    Attributes:
 9
10       y: array-like, shape = [n_samples]
11           Target values.
12
13       row_sample: double
14           subsampling fraction
15
16       n_samples: int
17            subsampling by using the number of rows
18
19       seed: int
20           reproductibility seed
21
22       n_jobs: int
23            number of jobs to run in parallel
24
25       verbose: bool
26            print progress messages and bars
27
28    Returns:
29
30        indices of subsampled y
31
32    """
33
34    def __init__(
35        self,
36        y,
37        row_sample=0.8,
38        n_samples=None,
39        seed=123,
40        n_jobs=None,
41        verbose=False,
42    ):
43        self.y = y
44        self.row_sample = row_sample
45        self.n_samples = n_samples
46        self.seed = seed
47        self.indices = None
48        self.n_jobs = n_jobs
49        self.verbose = verbose
50
51    def subsample(self):
52        self.indices = dosubsample(
53            self.y,
54            self.row_sample,
55            self.n_samples,
56            self.seed,
57            self.n_jobs,
58            self.verbose,
59        )
60        return self.indices

Subsampling class.

Attributes:

y: array-like, shape = [n_samples] Target values.

row_sample: double subsampling fraction

n_samples: int subsampling by using the number of rows

seed: int reproductibility seed

n_jobs: int number of jobs to run in parallel

verbose: bool print progress messages and bars

Returns:

indices of subsampled y
SubSampler( y, row_sample=0.8, n_samples=None, seed=123, n_jobs=None, verbose=False)
34    def __init__(
35        self,
36        y,
37        row_sample=0.8,
38        n_samples=None,
39        seed=123,
40        n_jobs=None,
41        verbose=False,
42    ):
43        self.y = y
44        self.row_sample = row_sample
45        self.n_samples = n_samples
46        self.seed = seed
47        self.indices = None
48        self.n_jobs = n_jobs
49        self.verbose = verbose
y
row_sample
n_samples
seed
indices
n_jobs
verbose
def subsample(self):
51    def subsample(self):
52        self.indices = dosubsample(
53            self.y,
54            self.row_sample,
55            self.n_samples,
56            self.seed,
57            self.n_jobs,
58            self.verbose,
59        )
60        return self.indices